From 417933a1e9fc1de832c6541792ce8068356b93fd Mon Sep 17 00:00:00 2001 From: micahjmartin Date: Sat, 10 Feb 2024 22:11:51 -0500 Subject: [PATCH 01/10] Add headers and recursive depth to file_list --- tavern/tomes/file_list/main.eldritch | 69 ++++++++++++++++++++++------ tavern/tomes/file_list/metadata.yml | 4 ++ 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/tavern/tomes/file_list/main.eldritch b/tavern/tomes/file_list/main.eldritch index 84863be9f..1168dc74b 100644 --- a/tavern/tomes/file_list/main.eldritch +++ b/tavern/tomes/file_list/main.eldritch @@ -1,18 +1,61 @@ -def file_list(path): +usernfo = sys.get_user() +PERM_READ = 4 + +name = { + "Directory": "Dir", + "File": "File", + "Link": "Link" +} + +def can_read(f): + """Return true if the user can read this dir/file + """ + f_user = int(f["permissions"][-3]) # User byte + f_group = int(f["permissions"][-2]) # Group byte + f_world = int(f["permissions"][-1]) # World byte applies to everyone + + # Are we root? + root = usernfo["euid"]["uid"] == 0 + + # If the user isnt root and the user doesnt own the file, clear the user byte + if not root and f["owner"] not in (usernfo["euid"]["name"], usernfo["uid"]["name"]): + f_user = 0 + + # TODO: https://github.com/spellshift/realm/issues/570 + # Will NOT match any group other than primary until #570 is fixed + + # If the user isnt root and the group doesnt own the file, clear the group byte + if not root and f["group"] not in (str(usernfo["egid"]), str(usernfo["gid"])): + f_group = 0 + + if PERM_READ in (f_world & PERM_READ, f_group & PERM_READ, f_user & PERM_READ): + return True + + return False + +def file_list(path, recurse=1, ignore_dirs=[]): + """List all files in the given path up to """ + # TODO: https://github.com/spellshift/realm/issues/569 + # No way to get perms of a path, so this will crash if the initial path is unreadable + # if not can_read(path): return + if file.is_dir(path): - files = file.list(path) - for f in files: - type_str = "" - if f['type'] == "Directory": - type_str = "Dir" - if f['type'] == "Link": - type_str = "Link" - if f['type'] == "File": - type_str = "File" - print(f['permissions']+"\t"+f['owner']+"\t"+f['group']+"\t"+str(f['size'])+"\t"+f['modified']+"\t"+type_str+"\t"+f['file_name']+"\n") + for f in file.list(path): + full = path.rstrip("/") + "/" + f["file_name"] + if f["type"] == "Directory": + full += "/" + print(f['permissions']+"\t"+f['owner']+"\t"+f['group']+"\t"+str(f['size'])+"\t"+f['modified']+"\t"+name.get(f['type'], f['type'])+"\t"+full+"\n") + if f["type"] == "Directory" and recurse-1 > 0: + # Skip files we cant read + if not can_read(f): continue + file_list(full, recurse=recurse-1, ignore_dirs=ignore_dirs) else: - print("Error: Invalid Path ("+path+")\n") + print("Error: Invalid Path ("+path+"), is a regular file\n") + +#input_params = {"path": "/tmp","depth": "0"} # Uncomment to test +# TODO: Is there any way to validate 'depth' is an int? -file_list(input_params['path']) +print("PERM\tUSER\tGROUP\tSIZE\tTIME\tTYPE\tPATH\n") +file_list(input_params['path'], recurse=int(input_params.get('depth', "1"))) print("\n") print("\n") diff --git a/tavern/tomes/file_list/metadata.yml b/tavern/tomes/file_list/metadata.yml index 65640f03d..bab59a049 100644 --- a/tavern/tomes/file_list/metadata.yml +++ b/tavern/tomes/file_list/metadata.yml @@ -8,3 +8,7 @@ paramdefs: type: string label: File path placeholder: "/etc/" +- name: depth + type: int + label: Recurse Depth + placeholder: "1" \ No newline at end of file From e0fc8f8861e0e9ec51939b448918b2a009f06b4d Mon Sep 17 00:00:00 2001 From: micahjmartin Date: Tue, 13 Feb 2024 15:45:15 -0500 Subject: [PATCH 02/10] remove recursion, add glob support --- tavern/tomes/file_list/main.eldritch | 109 ++++++++++++++++++++------- tavern/tomes/file_list/metadata.yml | 4 - 2 files changed, 80 insertions(+), 33 deletions(-) diff --git a/tavern/tomes/file_list/main.eldritch b/tavern/tomes/file_list/main.eldritch index 1168dc74b..6436e0b95 100644 --- a/tavern/tomes/file_list/main.eldritch +++ b/tavern/tomes/file_list/main.eldritch @@ -1,5 +1,4 @@ usernfo = sys.get_user() -PERM_READ = 4 name = { "Directory": "Dir", @@ -7,13 +6,21 @@ name = { "Link": "Link" } +SEP = "/" +if sys.get_os().get("platform", "") == "PLATFORM_WINDOWS": + SEP = "\\" + def can_read(f): """Return true if the user can read this dir/file """ + PERM_READ = 4 f_user = int(f["permissions"][-3]) # User byte f_group = int(f["permissions"][-2]) # Group byte - f_world = int(f["permissions"][-1]) # World byte applies to everyone - + + # Check world byte first so it hopefully is fast + if int(f["permissions"][-1]) & PERM_READ: + return True + # Are we root? root = usernfo["euid"]["uid"] == 0 @@ -28,34 +35,78 @@ def can_read(f): if not root and f["group"] not in (str(usernfo["egid"]), str(usernfo["gid"])): f_group = 0 - if PERM_READ in (f_world & PERM_READ, f_group & PERM_READ, f_user & PERM_READ): + if (f_group & PERM_READ) | (f_user & PERM_READ): return True - return False -def file_list(path, recurse=1, ignore_dirs=[]): - """List all files in the given path up to """ - # TODO: https://github.com/spellshift/realm/issues/569 - # No way to get perms of a path, so this will crash if the initial path is unreadable - # if not can_read(path): return - - if file.is_dir(path): - for f in file.list(path): - full = path.rstrip("/") + "/" + f["file_name"] - if f["type"] == "Directory": - full += "/" - print(f['permissions']+"\t"+f['owner']+"\t"+f['group']+"\t"+str(f['size'])+"\t"+f['modified']+"\t"+name.get(f['type'], f['type'])+"\t"+full+"\n") - if f["type"] == "Directory" and recurse-1 > 0: - # Skip files we cant read - if not can_read(f): continue - file_list(full, recurse=recurse-1, ignore_dirs=ignore_dirs) - else: - print("Error: Invalid Path ("+path+"), is a regular file\n") - -#input_params = {"path": "/tmp","depth": "0"} # Uncomment to test -# TODO: Is there any way to validate 'depth' is an int? - -print("PERM\tUSER\tGROUP\tSIZE\tTIME\tTYPE\tPATH\n") -file_list(input_params['path'], recurse=int(input_params.get('depth', "1"))) +def glob(s, pattern): + #if pattern == s or pattern == "*": + # return True + p = pattern.split("*") + # Check the first chunk + chunk = p.pop(0) + if not s.startswith(chunk): + return False + s = s[len(chunk):] + # Check the last chunk + if p: + chunk = p.pop() + if chunk: + if not s.endswith(chunk): + return False + s = s[:-len(chunk)] + + # Check all the middle chunks + for part in p: + if part not in s: + return False + s = s[s.index(part)+1:] + return True + +def print_file(path, f): + full = path.rstrip("/") + "/" + f["file_name"] + if f["type"] == "Directory": + full += "/" + print(f['permissions']+"\t"+f['owner']+"\t"+f['group']+"\t"+str(f['size'])+"\t"+f['modified']+"\t"+name.get(f['type'], f['type'])+"\t"+full+"\n") + +def file_list(path): + """List all files in the given path""" + parts = path.strip(SEP).split(SEP) + base = [] # The base of the path that doesnt have a glob + pattern = "" + for p in parts: + if '*' in p: + pattern = p + break + base.append(p) + + base = SEP.join(base) + if SEP == "/": + base = "/" + base + + # Safety checking + if not file.exists(base): + print("Error Path '"+path+"' does not exist\n") + return + elif file.is_file(base): + print("Error Path '"+path+"' is a file\n") + return + + # TODO: No way to check if we can read base + for f in file.list(base): + if pattern == "*": + # List each file/dir in this folder + if f["type"] == "Directory" and can_read(f): + d = base+SEP+f["file_name"] + for f in file.list(d): + print_file(d, f) + elif not pattern: + # Just list each file/folder + print_file(base, f) + elif glob(f["file_name"], pattern): + # Only print files/folders that match the glob + print_file(base, f) + +file_list(input_params['path']) print("\n") print("\n") diff --git a/tavern/tomes/file_list/metadata.yml b/tavern/tomes/file_list/metadata.yml index bab59a049..65640f03d 100644 --- a/tavern/tomes/file_list/metadata.yml +++ b/tavern/tomes/file_list/metadata.yml @@ -8,7 +8,3 @@ paramdefs: type: string label: File path placeholder: "/etc/" -- name: depth - type: int - label: Recurse Depth - placeholder: "1" \ No newline at end of file From 4781f8d0504411a44cc7d7587f1b3b0cdee90178 Mon Sep 17 00:00:00 2001 From: micahjmartin Date: Tue, 13 Feb 2024 15:53:32 -0500 Subject: [PATCH 03/10] update metadata --- tavern/tomes/file_list/metadata.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tavern/tomes/file_list/metadata.yml b/tavern/tomes/file_list/metadata.yml index 65640f03d..b18e689b9 100644 --- a/tavern/tomes/file_list/metadata.yml +++ b/tavern/tomes/file_list/metadata.yml @@ -1,5 +1,5 @@ name: List files -description: List the files and directories found at the path +description: List the files and directories found at the path. Supports basic glob functionality. Does not glob more than one level author: hulto support_model: FIRST_PARTY tactic: RECON From 996584e58c4172f0a62bd095376cf1d6f8d7fb53 Mon Sep 17 00:00:00 2001 From: micahjmartin Date: Tue, 13 Feb 2024 16:13:40 -0500 Subject: [PATCH 04/10] Add permissions check to file_tree --- tavern/tomes/file_tree/main.eldritch | 47 +++++++++++++++++++++++++--- tavern/tomes/file_tree/metadata.yml | 4 +++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/tavern/tomes/file_tree/main.eldritch b/tavern/tomes/file_tree/main.eldritch index 8b807f59d..b533e87a3 100644 --- a/tavern/tomes/file_tree/main.eldritch +++ b/tavern/tomes/file_tree/main.eldritch @@ -1,8 +1,40 @@ block_list = ["/proc","/sys","/lib","/libx32","/lib32","/lib64","/boot","/srv","/usr","/snap","/run","/dev","/cores"] +usernfo = sys.get_user() -def file_list(path,tree): +def can_read(f): + """Return true if the user can read this dir/file + """ + PERM_READ = 4 + f_user = int(f["permissions"][-3]) # User byte + f_group = int(f["permissions"][-2]) # Group byte + + # Check world byte first so it hopefully is fast + if int(f["permissions"][-1]) & PERM_READ: + return True + + # Are we root? + root = usernfo["euid"]["uid"] == 0 + + # If the user isnt root and the user doesnt own the file, clear the user byte + if not root and f["owner"] not in (usernfo["euid"]["name"], usernfo["uid"]["name"]): + f_user = 0 + + # TODO: https://github.com/spellshift/realm/issues/570 + # Will NOT match any group other than primary until #570 is fixed + + # If the user isnt root and the group doesnt own the file, clear the group byte + if not root and f["group"] not in (str(usernfo["egid"]), str(usernfo["gid"])): + f_group = 0 + + if (f_group & PERM_READ) | (f_user & PERM_READ): + return True + return False + +def file_list(path,tree, depth=-1): tree="|\t"+tree + if depth == 0: + return files = file.list(path) for f in files: if path+f['file_name'] in block_list: @@ -10,24 +42,29 @@ def file_list(path,tree): continue if f['type'] == "Directory": print(tree+"|---"+path+"/"+f['file_name']+"\n") - file_list(path+"/"+f['file_name'],tree) + if can_read(f): + file_list(path+"/"+f['file_name'],tree, depth=depth-1) if f['type'] == "Link": print(tree+"|---"+f['file_name']+"\n") if f['type'] == "File": print(tree+"|---"+f['file_name']+"\n") -def main(path): +def main(path, depth=-1): tree="" + if isinstance(depth, str) and not depth.lstrip("-").isdigit(): + print("ERROR: Invalid depth specified, must be a valid number") + return + depth=int(depth) if file.is_dir(path): print(path+"\n") if path == "/": print("It looks like you're trying to list every file on the system.\n") print("This generates a lot of data so I'm going to exclude less helpful directories\n") print("If you really really want everything including /proc and /sys specify \"//\"\n") - file_list(path,tree) + file_list(path,tree, depth=depth) elif file.is_file(path): print("Error: Invalid Path ("+path+")\n") -main(input_params['path']) +main(input_params['path'], input_params["depth"]) print("\n") print("\n") diff --git a/tavern/tomes/file_tree/metadata.yml b/tavern/tomes/file_tree/metadata.yml index 5e42bd14b..16c251b92 100644 --- a/tavern/tomes/file_tree/metadata.yml +++ b/tavern/tomes/file_tree/metadata.yml @@ -8,3 +8,7 @@ paramdefs: type: string label: File path placeholder: "/etc/" +- name: depth + type: int + label: Recurse Depth + placeholder: "-1" From a07260a73fd623634e7d49471fdadd940ae9c285 Mon Sep 17 00:00:00 2001 From: micahjmartin Date: Tue, 13 Feb 2024 21:45:31 -0500 Subject: [PATCH 05/10] Undo file_list changes --- tavern/tomes/file_list/main.eldritch | 120 +++------------------------ 1 file changed, 13 insertions(+), 107 deletions(-) diff --git a/tavern/tomes/file_list/main.eldritch b/tavern/tomes/file_list/main.eldritch index 6436e0b95..84863be9f 100644 --- a/tavern/tomes/file_list/main.eldritch +++ b/tavern/tomes/file_list/main.eldritch @@ -1,111 +1,17 @@ -usernfo = sys.get_user() - -name = { - "Directory": "Dir", - "File": "File", - "Link": "Link" -} - -SEP = "/" -if sys.get_os().get("platform", "") == "PLATFORM_WINDOWS": - SEP = "\\" - -def can_read(f): - """Return true if the user can read this dir/file - """ - PERM_READ = 4 - f_user = int(f["permissions"][-3]) # User byte - f_group = int(f["permissions"][-2]) # Group byte - - # Check world byte first so it hopefully is fast - if int(f["permissions"][-1]) & PERM_READ: - return True - - # Are we root? - root = usernfo["euid"]["uid"] == 0 - - # If the user isnt root and the user doesnt own the file, clear the user byte - if not root and f["owner"] not in (usernfo["euid"]["name"], usernfo["uid"]["name"]): - f_user = 0 - - # TODO: https://github.com/spellshift/realm/issues/570 - # Will NOT match any group other than primary until #570 is fixed - - # If the user isnt root and the group doesnt own the file, clear the group byte - if not root and f["group"] not in (str(usernfo["egid"]), str(usernfo["gid"])): - f_group = 0 - - if (f_group & PERM_READ) | (f_user & PERM_READ): - return True - return False - -def glob(s, pattern): - #if pattern == s or pattern == "*": - # return True - p = pattern.split("*") - # Check the first chunk - chunk = p.pop(0) - if not s.startswith(chunk): - return False - s = s[len(chunk):] - # Check the last chunk - if p: - chunk = p.pop() - if chunk: - if not s.endswith(chunk): - return False - s = s[:-len(chunk)] - - # Check all the middle chunks - for part in p: - if part not in s: - return False - s = s[s.index(part)+1:] - return True - -def print_file(path, f): - full = path.rstrip("/") + "/" + f["file_name"] - if f["type"] == "Directory": - full += "/" - print(f['permissions']+"\t"+f['owner']+"\t"+f['group']+"\t"+str(f['size'])+"\t"+f['modified']+"\t"+name.get(f['type'], f['type'])+"\t"+full+"\n") - def file_list(path): - """List all files in the given path""" - parts = path.strip(SEP).split(SEP) - base = [] # The base of the path that doesnt have a glob - pattern = "" - for p in parts: - if '*' in p: - pattern = p - break - base.append(p) - - base = SEP.join(base) - if SEP == "/": - base = "/" + base - - # Safety checking - if not file.exists(base): - print("Error Path '"+path+"' does not exist\n") - return - elif file.is_file(base): - print("Error Path '"+path+"' is a file\n") - return - - # TODO: No way to check if we can read base - for f in file.list(base): - if pattern == "*": - # List each file/dir in this folder - if f["type"] == "Directory" and can_read(f): - d = base+SEP+f["file_name"] - for f in file.list(d): - print_file(d, f) - elif not pattern: - # Just list each file/folder - print_file(base, f) - elif glob(f["file_name"], pattern): - # Only print files/folders that match the glob - print_file(base, f) + if file.is_dir(path): + files = file.list(path) + for f in files: + type_str = "" + if f['type'] == "Directory": + type_str = "Dir" + if f['type'] == "Link": + type_str = "Link" + if f['type'] == "File": + type_str = "File" + print(f['permissions']+"\t"+f['owner']+"\t"+f['group']+"\t"+str(f['size'])+"\t"+f['modified']+"\t"+type_str+"\t"+f['file_name']+"\n") + else: + print("Error: Invalid Path ("+path+")\n") file_list(input_params['path']) print("\n") From af79a3b72deed0c7b8b76e2788b4969677654bda Mon Sep 17 00:00:00 2001 From: micahjmartin Date: Tue, 13 Feb 2024 21:50:16 -0500 Subject: [PATCH 06/10] Clarifying placeholder --- tavern/tomes/file_tree/metadata.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tavern/tomes/file_tree/metadata.yml b/tavern/tomes/file_tree/metadata.yml index 16c251b92..0ee0a4c70 100644 --- a/tavern/tomes/file_tree/metadata.yml +++ b/tavern/tomes/file_tree/metadata.yml @@ -11,4 +11,4 @@ paramdefs: - name: depth type: int label: Recurse Depth - placeholder: "-1" + placeholder: "3, or -1 for infinite recursion" From 10293fb2a6057a1130a4e749af69cc8b510a15b5 Mon Sep 17 00:00:00 2001 From: micahjmartin Date: Tue, 13 Feb 2024 21:58:19 -0500 Subject: [PATCH 07/10] Missed a line, rip build system --- tavern/tomes/file_list/metadata.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tavern/tomes/file_list/metadata.yml b/tavern/tomes/file_list/metadata.yml index b18e689b9..65640f03d 100644 --- a/tavern/tomes/file_list/metadata.yml +++ b/tavern/tomes/file_list/metadata.yml @@ -1,5 +1,5 @@ name: List files -description: List the files and directories found at the path. Supports basic glob functionality. Does not glob more than one level +description: List the files and directories found at the path author: hulto support_model: FIRST_PARTY tactic: RECON From 0a85dfe533631fc1a58f4ac1dba708acfe5a12e9 Mon Sep 17 00:00:00 2001 From: micahjmartin Date: Fri, 16 Feb 2024 00:35:17 -0500 Subject: [PATCH 08/10] Bypass windows permissions --- tavern/tomes/file_tree/main.eldritch | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tavern/tomes/file_tree/main.eldritch b/tavern/tomes/file_tree/main.eldritch index b533e87a3..2bbe16510 100644 --- a/tavern/tomes/file_tree/main.eldritch +++ b/tavern/tomes/file_tree/main.eldritch @@ -1,10 +1,15 @@ block_list = ["/proc","/sys","/lib","/libx32","/lib32","/lib64","/boot","/srv","/usr","/snap","/run","/dev","/cores"] usernfo = sys.get_user() +windows = sys.get_os().get("platform", "") == "PLATFORM_WINDOWS" def can_read(f): """Return true if the user can read this dir/file """ + # Bypass until windows perms are implemented + if windows: + return True + PERM_READ = 4 f_user = int(f["permissions"][-3]) # User byte f_group = int(f["permissions"][-2]) # Group byte From a600ba555880c5c43699aeff657d91d6b883da19 Mon Sep 17 00:00:00 2001 From: micahjmartin Date: Mon, 26 Feb 2024 20:27:04 -0500 Subject: [PATCH 09/10] Fix windows bug and strip newlines --- tavern/tomes/file_tree/main.eldritch | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tavern/tomes/file_tree/main.eldritch b/tavern/tomes/file_tree/main.eldritch index 2bbe16510..555c98c3f 100644 --- a/tavern/tomes/file_tree/main.eldritch +++ b/tavern/tomes/file_tree/main.eldritch @@ -1,7 +1,7 @@ block_list = ["/proc","/sys","/lib","/libx32","/lib32","/lib64","/boot","/srv","/usr","/snap","/run","/dev","/cores"] usernfo = sys.get_user() -windows = sys.get_os().get("platform", "") == "PLATFORM_WINDOWS" +windows = sys.is_windows() def can_read(f): """Return true if the user can read this dir/file @@ -43,16 +43,16 @@ def file_list(path,tree, depth=-1): files = file.list(path) for f in files: if path+f['file_name'] in block_list: - print("Skipping: "+path+f['file_name']+"\n") + print("Skipping: "+path+f['file_name']) continue if f['type'] == "Directory": - print(tree+"|---"+path+"/"+f['file_name']+"\n") + print(tree+"|---"+f['file_name']+"/") if can_read(f): file_list(path+"/"+f['file_name'],tree, depth=depth-1) if f['type'] == "Link": - print(tree+"|---"+f['file_name']+"\n") + print(tree+"|---"+f['file_name']) if f['type'] == "File": - print(tree+"|---"+f['file_name']+"\n") + print(tree+"|---"+f['file_name']) def main(path, depth=-1): tree="" @@ -63,13 +63,11 @@ def main(path, depth=-1): if file.is_dir(path): print(path+"\n") if path == "/": - print("It looks like you're trying to list every file on the system.\n") - print("This generates a lot of data so I'm going to exclude less helpful directories\n") - print("If you really really want everything including /proc and /sys specify \"//\"\n") + print("It looks like you're trying to list every file on the system.") + print("This generates a lot of data so I'm going to exclude less helpful directories") + print("If you really really want everything including /proc and /sys specify \"//\"") file_list(path,tree, depth=depth) elif file.is_file(path): - print("Error: Invalid Path ("+path+")\n") + print("Error: Invalid Path ("+path+")") -main(input_params['path'], input_params["depth"]) -print("\n") -print("\n") +main(input_params['path'], input_params["depth"]) \ No newline at end of file From fdd8d6091d22f78c17c119b31ea563cdd939d779 Mon Sep 17 00:00:00 2001 From: micahjmartin Date: Mon, 26 Feb 2024 22:30:52 -0500 Subject: [PATCH 10/10] remove depth check --- tavern/tomes/file_tree/main.eldritch | 3 --- 1 file changed, 3 deletions(-) diff --git a/tavern/tomes/file_tree/main.eldritch b/tavern/tomes/file_tree/main.eldritch index 555c98c3f..0885f4dbc 100644 --- a/tavern/tomes/file_tree/main.eldritch +++ b/tavern/tomes/file_tree/main.eldritch @@ -56,9 +56,6 @@ def file_list(path,tree, depth=-1): def main(path, depth=-1): tree="" - if isinstance(depth, str) and not depth.lstrip("-").isdigit(): - print("ERROR: Invalid depth specified, must be a valid number") - return depth=int(depth) if file.is_dir(path): print(path+"\n")