From a03f6ce28d0ac5dc56a475bf99bbd7529c2543b4 Mon Sep 17 00:00:00 2001 From: nullableVoidPtr <30564701+nullableVoidPtr@users.noreply.github.com> Date: Sat, 7 May 2022 17:46:42 +0800 Subject: [PATCH 1/7] Remove nonexistent syspage.bin from MANIFEST --- MANIFEST.in | 1 - 1 file changed, 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index ead0863c9..3a680e281 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,4 +2,3 @@ recursive-include qiling/debugger/gdb/xml * recursive-include qiling/extensions/windows_sdk/defs * recursive-include qiling/profiles * include qiling/os/uefi/guids.csv -include qiling/os/qnx/syspage.bin From d3336291fe8df40be468b25bec46d1eb80a4b9e0 Mon Sep 17 00:00:00 2001 From: nullableVoidPtr <30564701+nullableVoidPtr@users.noreply.github.com> Date: Sat, 7 May 2022 19:17:30 +0800 Subject: [PATCH 2/7] Use pkgutil instead of absolute paths --- MANIFEST.in | 1 + qiling/arch/evm/analysis/signatures.py | 7 +++---- qiling/debugger/gdb/gdb.py | 17 +++++++++-------- qiling/os/uefi/__init__.py | 11 ++++------- qiling/utils.py | 17 +++++++---------- 5 files changed, 24 insertions(+), 29 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 3a680e281..295f846d0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,3 +2,4 @@ recursive-include qiling/debugger/gdb/xml * recursive-include qiling/extensions/windows_sdk/defs * recursive-include qiling/profiles * include qiling/os/uefi/guids.csv +include qiling/arch/evm/analysis/signatures.json diff --git a/qiling/arch/evm/analysis/signatures.py b/qiling/arch/evm/analysis/signatures.py index aebe55f0d..321260fe9 100644 --- a/qiling/arch/evm/analysis/signatures.py +++ b/qiling/arch/evm/analysis/signatures.py @@ -1,4 +1,4 @@ -import os +import pkgutil import re import logging import json @@ -92,9 +92,8 @@ def analysis_func_sign(insns:list, engine_num=1): class signatures_engine_1: @staticmethod def find_signature(sign): - path = os.path.split(os.path.realpath(__file__))[0] + '/signatures.json' - with open(path) as data_file: - data = json.load(data_file) + data = pkgutil.get_data(__package__, 'signatures.json').decode() + data = json.load(data) list_name = [name for name, hexa in data.items() if hexa == sign] diff --git a/qiling/debugger/gdb/gdb.py b/qiling/debugger/gdb/gdb.py index 8b3416663..6bccf5b0c 100644 --- a/qiling/debugger/gdb/gdb.py +++ b/qiling/debugger/gdb/gdb.py @@ -7,6 +7,7 @@ # documentation: according to https://sourceware.org/gdb/current/onlinedocs/gdb/Remote-Protocol.html#Remote-Protocol import struct, os, socket +import pkgutil from binascii import unhexlify from typing import Iterator, Literal @@ -481,15 +482,15 @@ def handle_q(subcmd): else: self.send("PacketSize=47ff;QPassSignals+;QProgramSignals+;QStartupWithShell+;QEnvironmentHexEncoded+;QEnvironmentReset+;QEnvironmentUnset+;QSetWorkingDir+;QCatchSyscalls+;qXfer:libraries-svr4:read+;augmented-libraries-svr4-read+;qXfer:auxv:read+;qXfer:siginfo:read+;qXfer:siginfo:write+;qXfer:features:read+;QStartNoAckMode+;qXfer:osdata:read+;multiprocess+;fork-events+;vfork-events+;exec-events+;QNonStop+;QDisableRandomization+;qXfer:threads:read+;ConditionalTracepoints+;TraceStateVariables+;TracepointSource+;DisconnectedTracing+;FastTracepoints+;StaticTracepoints+;InstallInTrace+;qXfer:statictrace:read+;qXfer:traceframe-info:read+;EnableDisableTracepoints+;QTBuffer:size+;tracenz+;ConditionalBreakpoints+;BreakpointCommands+;QAgent+;Qbtrace:bts+;Qbtrace-conf:bts:size+;Qbtrace:pt+;Qbtrace-conf:pt:size+;Qbtrace:off+;qXfer:btrace:read+;qXfer:btrace-conf:read+;swbreak+;hwbreak+;qXfer:exec-file:read+;vContSupported+;QThreadEvents+;no-resumed+") elif subcmd.startswith('Xfer:features:read'): - xfercmd_file = subcmd.split(':')[3] - xfercmd_abspath = os.path.dirname(os.path.abspath(__file__)) - xml_folder = self.ql.arch.type.name.lower() - xfercmd_file = os.path.join(xfercmd_abspath,"xml",xml_folder, xfercmd_file) - - if os.path.exists(xfercmd_file) and self.ql.os.type is not QL_OS.WINDOWS: - with open(xfercmd_file, 'r') as f: - file_contents = f.read() + if self.ql.os.type is not QL_OS.WINDOWS: + try: + xfercmd_file = subcmd.split(':')[3] + xml_folder = self.ql.arch.type.name.lower() + file_contents = pkgutil.get_data(__package__, f"xml/{xml_folder}/{xfercmd_file}").decode() self.send("l%s" % file_contents) + except: + self.ql.log.info("gdb> Platform is not supported by xml or xml file not found: %s\n" % (xfercmd_file)) + self.send("l") else: self.ql.log.info("gdb> Platform is not supported by xml or xml file not found: %s\n" % (xfercmd_file)) self.send("l") diff --git a/qiling/os/uefi/__init__.py b/qiling/os/uefi/__init__.py index 5b8faeab6..aa3902544 100644 --- a/qiling/os/uefi/__init__.py +++ b/qiling/os/uefi/__init__.py @@ -1,17 +1,14 @@ import csv from typing import Mapping -from os import path +import pkgutil def __init_guids_db() -> Mapping[str, str]: """Initialize GUIDs dictionary from a local database. """ - csv_path = path.dirname(path.abspath(__file__)) - csv_path = path.join(csv_path, 'guids.csv') + guids_file = pkgutil.get_data(__package__, 'guids.csv').decode() + guids_reader = csv.reader(guids_file.splitlines()) - with open(csv_path) as guids_file: - guids_reader = csv.reader(guids_file) - - return dict(tuple(entry) for entry in guids_reader) + return dict(tuple(entry) for entry in guids_reader) guids_db = __init_guids_db() diff --git a/qiling/utils.py b/qiling/utils.py index 68607fe60..b3180ed1a 100644 --- a/qiling/utils.py +++ b/qiling/utils.py @@ -9,7 +9,7 @@ """ from functools import partial -import importlib, os +import importlib, pkgutil, os from configparser import ConfigParser from types import ModuleType @@ -415,19 +415,16 @@ def profile_setup(ostype: QL_OS, filename: Optional[str]): config = {} else: - qiling_home = os.path.dirname(os.path.abspath(__file__)) - os_profile = os.path.join(qiling_home, 'profiles', f'{ostype.name.lower()}.ql') - - profiles = [os_profile] - - if filename: - profiles.append(filename) - # patch 'getint' to convert integers of all bases int_converter = partial(int, base=0) config = ConfigParser(converters={'int': int_converter}) - config.read(profiles) + + os_profile = pkgutil.get_data(__package__, f'profiles/{ostype.name.lower()}.ql').decode() + config.read_string(os_profile) + + if filename: + config.read(filename) return config From 84798b19455f24c3cebce114d877e5b748ccecd9 Mon Sep 17 00:00:00 2001 From: nullableVoidPtr <30564701+nullableVoidPtr@users.noreply.github.com> Date: Sat, 7 May 2022 19:45:07 +0800 Subject: [PATCH 3/7] Fix for nonexistent profiles --- qiling/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/qiling/utils.py b/qiling/utils.py index b3180ed1a..09bb68a32 100644 --- a/qiling/utils.py +++ b/qiling/utils.py @@ -420,8 +420,11 @@ def profile_setup(ostype: QL_OS, filename: Optional[str]): config = ConfigParser(converters={'int': int_converter}) - os_profile = pkgutil.get_data(__package__, f'profiles/{ostype.name.lower()}.ql').decode() - config.read_string(os_profile) + try: + os_profile = pkgutil.get_data(__package__, f'profiles/{ostype.name.lower()}.ql').decode() + config.read_string(os_profile) + except FileNotFoundError as e: + pass if filename: config.read(filename) From d166a984cdef163c3809847afb48f472a1ddcf42 Mon Sep 17 00:00:00 2001 From: nullableVoidPtr <30564701+nullableVoidPtr@users.noreply.github.com> Date: Sat, 7 May 2022 19:52:48 +0800 Subject: [PATCH 4/7] Run giteesync only on main repository --- .github/workflows/giteesync.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/giteesync.yml b/.github/workflows/giteesync.yml index 4e7104ba4..dbf797497 100644 --- a/.github/workflows/giteesync.yml +++ b/.github/workflows/giteesync.yml @@ -5,6 +5,7 @@ jobs: deploy: runs-on: ubuntu-latest + if: github.repository_owner == 'qilingframework' steps: - uses: actions/checkout@v2 with: From d051174810e716936f27831144c2a43ceaf9844b Mon Sep 17 00:00:00 2001 From: nullableVoidPtr <30564701+nullableVoidPtr@users.noreply.github.com> Date: Sat, 7 May 2022 20:14:00 +0800 Subject: [PATCH 5/7] Replace MANIFEST.in with package_data --- MANIFEST.in | 5 ----- setup.py | 7 ++++++- 2 files changed, 6 insertions(+), 6 deletions(-) delete mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 295f846d0..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -recursive-include qiling/debugger/gdb/xml * -recursive-include qiling/extensions/windows_sdk/defs * -recursive-include qiling/profiles * -include qiling/os/uefi/guids.csv -include qiling/arch/evm/analysis/signatures.json diff --git a/setup.py b/setup.py index 5496da6cd..3987cc252 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,12 @@ packages=find_packages(), scripts=['qltool'], - include_package_data=True, + package_data={ + 'qiling': ['profiles/*.ql'], + 'qiling.debugger.gdb': ['xml/*/*'], + 'qiling.os.uefi': ['guids.csv'], + 'qiling.arch.evm.analysis': ['signatures.json'] + }, install_requires=requirements, extras_require=extras, ) From 719b98a61c6a996916f7982b09aac8514944616a Mon Sep 17 00:00:00 2001 From: nullableVoidPtr <30564701+nullableVoidPtr@users.noreply.github.com> Date: Sat, 7 May 2022 20:22:39 +0800 Subject: [PATCH 6/7] Add to credits --- CREDITS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CREDITS.md b/CREDITS.md index b4e32331e..9ea9f0edf 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -54,6 +54,7 @@ - danielmoos - sigeryang - bet4it +- nullableVoidPtr #### Legacy Core Developers From 9c6140de0b151afb1e7c4e80b4053474e16d0f43 Mon Sep 17 00:00:00 2001 From: nullableVoidPtr <30564701+nullableVoidPtr@users.noreply.github.com> Date: Sun, 8 May 2022 18:44:05 +0800 Subject: [PATCH 7/7] pkgutil -> inspect + Path As per elicn's comments. TODO: If Qiling allows 3.9 standard Libs, move everything to importlib.resources.files --- qiling/arch/evm/analysis/signatures.py | 8 +++++--- qiling/debugger/gdb/gdb.py | 20 ++++++++++---------- qiling/os/uefi/__init__.py | 11 +++++++---- qiling/utils.py | 21 +++++++++++---------- 4 files changed, 33 insertions(+), 27 deletions(-) diff --git a/qiling/arch/evm/analysis/signatures.py b/qiling/arch/evm/analysis/signatures.py index 321260fe9..e7eed5092 100644 --- a/qiling/arch/evm/analysis/signatures.py +++ b/qiling/arch/evm/analysis/signatures.py @@ -1,4 +1,5 @@ -import pkgutil +import inspect +from pathlib import Path import re import logging import json @@ -92,8 +93,9 @@ def analysis_func_sign(insns:list, engine_num=1): class signatures_engine_1: @staticmethod def find_signature(sign): - data = pkgutil.get_data(__package__, 'signatures.json').decode() - data = json.load(data) + path = Path(inspect.getfile(inspect.getframe())).parent / 'signatures.json' + with path.open('r') as data_file: + data = json.load(data_file) list_name = [name for name, hexa in data.items() if hexa == sign] diff --git a/qiling/debugger/gdb/gdb.py b/qiling/debugger/gdb/gdb.py index 6bccf5b0c..768e2e0a8 100644 --- a/qiling/debugger/gdb/gdb.py +++ b/qiling/debugger/gdb/gdb.py @@ -7,7 +7,8 @@ # documentation: according to https://sourceware.org/gdb/current/onlinedocs/gdb/Remote-Protocol.html#Remote-Protocol import struct, os, socket -import pkgutil +import inspect +from pathlib import Path from binascii import unhexlify from typing import Iterator, Literal @@ -482,15 +483,14 @@ def handle_q(subcmd): else: self.send("PacketSize=47ff;QPassSignals+;QProgramSignals+;QStartupWithShell+;QEnvironmentHexEncoded+;QEnvironmentReset+;QEnvironmentUnset+;QSetWorkingDir+;QCatchSyscalls+;qXfer:libraries-svr4:read+;augmented-libraries-svr4-read+;qXfer:auxv:read+;qXfer:siginfo:read+;qXfer:siginfo:write+;qXfer:features:read+;QStartNoAckMode+;qXfer:osdata:read+;multiprocess+;fork-events+;vfork-events+;exec-events+;QNonStop+;QDisableRandomization+;qXfer:threads:read+;ConditionalTracepoints+;TraceStateVariables+;TracepointSource+;DisconnectedTracing+;FastTracepoints+;StaticTracepoints+;InstallInTrace+;qXfer:statictrace:read+;qXfer:traceframe-info:read+;EnableDisableTracepoints+;QTBuffer:size+;tracenz+;ConditionalBreakpoints+;BreakpointCommands+;QAgent+;Qbtrace:bts+;Qbtrace-conf:bts:size+;Qbtrace:pt+;Qbtrace-conf:pt:size+;Qbtrace:off+;qXfer:btrace:read+;qXfer:btrace-conf:read+;swbreak+;hwbreak+;qXfer:exec-file:read+;vContSupported+;QThreadEvents+;no-resumed+") elif subcmd.startswith('Xfer:features:read'): - if self.ql.os.type is not QL_OS.WINDOWS: - try: - xfercmd_file = subcmd.split(':')[3] - xml_folder = self.ql.arch.type.name.lower() - file_contents = pkgutil.get_data(__package__, f"xml/{xml_folder}/{xfercmd_file}").decode() - self.send("l%s" % file_contents) - except: - self.ql.log.info("gdb> Platform is not supported by xml or xml file not found: %s\n" % (xfercmd_file)) - self.send("l") + xfercmd_file = subcmd.split(':')[3] + xfercmd_abspath = Path(inspect.getfile(inspect.currentframe())).parent + xml_folder = self.ql.arch.type.name.lower() + xfercmd_file = xfercmd_abspath / 'xml' / xml_folder / xfercmd_file + + if xfercmd_file.exists() and self.ql.os.type is not QL_OS.WINDOWS: + with xfercmd_file.open('r') as f: + file_contents = f.read() else: self.ql.log.info("gdb> Platform is not supported by xml or xml file not found: %s\n" % (xfercmd_file)) self.send("l") diff --git a/qiling/os/uefi/__init__.py b/qiling/os/uefi/__init__.py index aa3902544..0e68ec3c5 100644 --- a/qiling/os/uefi/__init__.py +++ b/qiling/os/uefi/__init__.py @@ -1,14 +1,17 @@ import csv from typing import Mapping -import pkgutil +import inspect +from pathlib import Path def __init_guids_db() -> Mapping[str, str]: """Initialize GUIDs dictionary from a local database. """ - guids_file = pkgutil.get_data(__package__, 'guids.csv').decode() - guids_reader = csv.reader(guids_file.splitlines()) + csv_path = Path(inspect.getfile(inspect.currentframe())).parent / 'guids.csv' - return dict(tuple(entry) for entry in guids_reader) + with csv_path.open('r') as guids_file: + guids_reader = csv.reader(guids_file) + + return dict(tuple(entry) for entry in guids_reader) guids_db = __init_guids_db() diff --git a/qiling/utils.py b/qiling/utils.py index 09bb68a32..dc3703052 100644 --- a/qiling/utils.py +++ b/qiling/utils.py @@ -9,7 +9,8 @@ """ from functools import partial -import importlib, pkgutil, os +from pathlib import Path +import importlib, inspect, os from configparser import ConfigParser from types import ModuleType @@ -415,19 +416,19 @@ def profile_setup(ostype: QL_OS, filename: Optional[str]): config = {} else: + qiling_home = Path(inspect.getfile(inspect.currentframe())).parent + os_profile = qiling_home / 'profiles' / f'{ostype.name.lower()}.ql' + + profiles = [os_profile] + + if filename: + profiles.append(filename) + # patch 'getint' to convert integers of all bases int_converter = partial(int, base=0) config = ConfigParser(converters={'int': int_converter}) - - try: - os_profile = pkgutil.get_data(__package__, f'profiles/{ostype.name.lower()}.ql').decode() - config.read_string(os_profile) - except FileNotFoundError as e: - pass - - if filename: - config.read(filename) + config.read(profiles) return config