From 1a502766a87572350beb5a22a6373fff8a89999d Mon Sep 17 00:00:00 2001 From: elicn Date: Sun, 23 Oct 2022 00:01:39 +0300 Subject: [PATCH 1/4] Improve handling of 42000 magic pid --- qiling/debugger/gdb/gdb.py | 42 +++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/qiling/debugger/gdb/gdb.py b/qiling/debugger/gdb/gdb.py index 040614458..4d8f16e17 100644 --- a/qiling/debugger/gdb/gdb.py +++ b/qiling/debugger/gdb/gdb.py @@ -14,9 +14,9 @@ # https://sourceware.org/gdb/current/onlinedocs/gdb/Remote-Protocol.html import os, socket, re, tempfile +from functools import partial from logging import Logger -from typing import Iterator, Mapping, Optional, Union -import typing +from typing import IO, Iterator, MutableMapping, Optional, Union from unicorn import UcError from unicorn.unicorn_const import ( @@ -32,7 +32,6 @@ from qiling.debugger.gdb.xmlregs import QlGdbFeatures from qiling.debugger.gdb.utils import QlGdbUtils from qiling.os.linux.procfs import QlProcFS -from qiling.os.mapper import QlFsMappedCallable, QlFsMappedObject # gdb logging prompt PROMPT = r'gdb>' @@ -104,10 +103,7 @@ def __init__(self, ql: Qiling, ip: str = '127.0.0.1', port: int = 9999): self.features = QlGdbFeatures(self.ql.arch.type, self.ql.os.type) self.regsmap = self.features.regsmap - # https://sourceware.org/bugzilla/show_bug.cgi?id=17760 - # 42000 is the magic pid to indicate the remote process. - self.ql.add_fs_mapper(r'/proc/42000/maps', QlFsMappedCallable(QlProcFS.self_map, self.ql.mem)) - self.fake_procfs: Mapping[int, typing.IO] = {} + self.fake_procfs: MutableMapping[int, IO] = {} def run(self): server = GdbSerialConn(self.ip, self.port, self.ql.log) @@ -590,15 +586,29 @@ def handle_v(subcmd: str) -> Reply: virtpath = self.ql.os.path.virtual_abspath(path) - if virtpath.startswith(r'/proc') and self.ql.os.fs_mapper.has_mapping(virtpath): - # Mapped object by itself is not backed with a host fd and thus a tempfile can - # 1. Make pread easy to implement and avoid duplicate code like seek, fd etc. - # 2. Avoid fd clash if we assign a generated fd. - tfile = tempfile.TemporaryFile("rb+") - tfile.write(self.ql.os.fs_mapper.open(virtpath, "rb+").read()) - tfile.seek(0, os.SEEK_SET) - fd = tfile.fileno() - self.fake_procfs[fd] = tfile + if virtpath.startswith(r'/proc/'): + pid, _, vfname = virtpath[6:].partition(r'/') + + # 42000 is a magic number indicating the remote process' pid + # see: https://sourceware.org/bugzilla/show_bug.cgi?id=17760 + if pid == '42000': + vfmap = { + 'maps': lambda: partial(QlProcFS.self_map, self.ql.mem) + } + + if vfname in vfmap and not self.ql.os.fs_mapper.has_mapping(virtpath): + self.ql.add_fs_mapper(virtpath, vfmap[vfname]()) + + if self.ql.os.fs_mapper.has_mapping(virtpath): + # Mapped object by itself is not backed with a host fd and thus a tempfile can + # 1. Make pread easy to implement and avoid duplicate code like seek, fd etc. + # 2. Avoid fd clash if we assign a generated fd. + tfile = tempfile.TemporaryFile("rb+") + tfile.write(self.ql.os.fs_mapper.open(virtpath, "rb+").read()) + tfile.seek(0, os.SEEK_SET) + + fd = tfile.fileno() + self.fake_procfs[fd] = tfile else: host_path = self.ql.os.path.virtual_to_host_path(path) From e7b281b744589c97a9c3b9adcad0beca365cbdb0 Mon Sep 17 00:00:00 2001 From: elicn Date: Sun, 23 Oct 2022 00:03:56 +0300 Subject: [PATCH 2/4] Replace QlFsMappedCallable with functools.partial --- qiling/os/linux/linux.py | 13 +++++++------ qiling/os/mapper.py | 19 +++---------------- 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/qiling/os/linux/linux.py b/qiling/os/linux/linux.py index 4198b2c35..ee9a2b06b 100644 --- a/qiling/os/linux/linux.py +++ b/qiling/os/linux/linux.py @@ -3,6 +3,8 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +from functools import partial + from unicorn import UcError from unicorn.x86_const import UC_X86_INS_SYSCALL @@ -15,7 +17,6 @@ from qiling.os.fcall import QlFunctionCall from qiling.os.const import * from qiling.os.linux.procfs import QlProcFS -from qiling.os.mapper import QlFsMappedCallable from qiling.os.posix.posix import QlOsPosix from . import futex @@ -122,11 +123,11 @@ def load(self): def setup_procfs(self): - self.fs_mapper.add_fs_mapping(r'/proc/self/auxv', QlFsMappedCallable(QlProcFS.self_auxv, self)) - self.fs_mapper.add_fs_mapping(r'/proc/self/cmdline', QlFsMappedCallable(QlProcFS.self_cmdline, self)) - self.fs_mapper.add_fs_mapping(r'/proc/self/environ', QlFsMappedCallable(QlProcFS.self_environ, self)) - self.fs_mapper.add_fs_mapping(r'/proc/self/exe', QlFsMappedCallable(QlProcFS.self_exe, self)) - self.fs_mapper.add_fs_mapping(r'/proc/self/maps', QlFsMappedCallable(QlProcFS.self_map, self.ql.mem)) + self.fs_mapper.add_fs_mapping(r'/proc/self/auxv', partial(QlProcFS.self_auxv, self)) + self.fs_mapper.add_fs_mapping(r'/proc/self/cmdline', partial(QlProcFS.self_cmdline, self)) + self.fs_mapper.add_fs_mapping(r'/proc/self/environ', partial(QlProcFS.self_environ, self)) + self.fs_mapper.add_fs_mapping(r'/proc/self/exe', partial(QlProcFS.self_exe, self)) + self.fs_mapper.add_fs_mapping(r'/proc/self/maps', partial(QlProcFS.self_map, self.ql.mem)) def hook_syscall(self, ql, intno = None): return self.load_syscall() diff --git a/qiling/os/mapper.py b/qiling/os/mapper.py index bbc226b99..111fb47ce 100644 --- a/qiling/os/mapper.py +++ b/qiling/os/mapper.py @@ -3,9 +3,8 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import inspect import os -from typing import Any, MutableMapping, Union +from typing import Any, Callable, MutableMapping, Union from .path import QlOsPath from .filestruct import ql_file @@ -57,19 +56,7 @@ def readline(self, end = b'\n'): def name(self): raise NotImplementedError("QlFsMappedObject property not implemented: name") -# This is a helper class to allow users to pass any class to add_fs_mapper -# -# Everytime open is called on the mapped path, cls(*args, **kwargs) will be called. -class QlFsMappedCallable: - - def __init__(self, cls, *args, **kwargs) -> None: - self._args = args - self._kwargs = kwargs - self._cls = cls - - def __call__(self) -> QlFsMappedObject: - return self._cls(*self._args, **self._kwargs) - + class QlFsMapper: def __init__(self, path: QlOsPath): @@ -144,7 +131,7 @@ def _parse_path(self, p: Union[os.PathLike, str]) -> str: return p - def add_fs_mapping(self, ql_path: Union[os.PathLike, str], real_dest: Union[str, QlFsMappedObject, QlFsMappedCallable]) -> None: + def add_fs_mapping(self, ql_path: Union[os.PathLike, str], real_dest: Union[str, QlFsMappedObject, Callable]) -> None: """Map an object to Qiling emulated file system. Args: From 92fb43497f74d4d873f9ad687281e0c7667a0f09 Mon Sep 17 00:00:00 2001 From: elicn Date: Sun, 23 Oct 2022 00:05:45 +0300 Subject: [PATCH 3/4] Improve gdb features advertisement --- qiling/debugger/gdb/gdb.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/qiling/debugger/gdb/gdb.py b/qiling/debugger/gdb/gdb.py index 4d8f16e17..efbacc103 100644 --- a/qiling/debugger/gdb/gdb.py +++ b/qiling/debugger/gdb/gdb.py @@ -392,13 +392,9 @@ def handle_q(subcmd: str) -> Reply: 'QAgent+', 'QCatchSyscalls+', 'QDisableRandomization+', - 'QEnvironmentHexEncoded+', - 'QEnvironmentReset+', - 'QEnvironmentUnset+', 'QNonStop+', 'QPassSignals+', 'QProgramSignals+', - 'QSetWorkingDir+', 'QStartNoAckMode+', 'QStartupWithShell+', 'QTBuffer:size+', @@ -411,7 +407,6 @@ def handle_q(subcmd: str) -> Reply: 'hwbreak+', 'multiprocess+', 'no-resumed+', - 'qXfer:auxv:read+', 'qXfer:features:read+', # 'qXfer:libraries-svr4:read+', # 'qXfer:osdata:read+', @@ -449,9 +444,19 @@ def handle_q(subcmd: str) -> Reply: # os dependent features if not self.ql.interpreter: + features += [ + 'QEnvironmentHexEncoded+', + 'QEnvironmentReset+', + 'QEnvironmentUnset+' + ] + # filesystem dependent features if hasattr(self.ql.os, 'path'): - features.append('qXfer:exec-file:read+') + features += [ + 'QSetWorkingDir+', + 'qXfer:auxv:read+', + 'qXfer:exec-file:read+' + ] # process dependent features if hasattr(self.ql.os, 'pid'): From 3030823b6833dcc453f91de509b3ee13ab5bc66d Mon Sep 17 00:00:00 2001 From: elicn Date: Sun, 23 Oct 2022 00:06:39 +0300 Subject: [PATCH 4/4] Opportunistic PEP8 and linter-friendly tweaks --- qiling/debugger/gdb/gdb.py | 32 ++++++++------------------------ qiling/os/linux/linux.py | 12 ++++-------- qiling/os/linux/procfs.py | 13 +++++-------- qiling/os/mapper.py | 22 +++++++++++----------- 4 files changed, 28 insertions(+), 51 deletions(-) diff --git a/qiling/debugger/gdb/gdb.py b/qiling/debugger/gdb/gdb.py index efbacc103..53e034d2e 100644 --- a/qiling/debugger/gdb/gdb.py +++ b/qiling/debugger/gdb/gdb.py @@ -13,7 +13,10 @@ # gdb remote protocol: # https://sourceware.org/gdb/current/onlinedocs/gdb/Remote-Protocol.html -import os, socket, re, tempfile +import os +import socket +import re +import tempfile from functools import partial from logging import Logger from typing import IO, Iterator, MutableMapping, Optional, Union @@ -61,6 +64,7 @@ # reply type Reply = Union[bytes, str] + class QlGdb(QlDebugger): """A simple gdbserver implementation. """ @@ -154,11 +158,9 @@ def __swap_endianess(value: int) -> int: return int.from_bytes(raw, 'big') - def handle_exclaim(subcmd: str) -> Reply: return REPLY_OK - def handle_qmark(subcmd: str) -> Reply: """Request status. @@ -207,7 +209,6 @@ def __get_reg_info(ucreg: int) -> str: return f'T{SIGTRAP:02x}{bp_info}{sp_info}{pc_info}' - def handle_c(subcmd: str) -> Reply: try: self.gdb.resume_emu() @@ -242,7 +243,6 @@ def handle_c(subcmd: str) -> Reply: return reply - def handle_g(subcmd: str) -> Reply: # NOTE: in the past the 'g' reply packet for arm included the f0-f7 and fps registers between pc # and cpsr, which placed cpsr at index (regnum) 25. as the f-registers became obsolete the cpsr @@ -257,7 +257,6 @@ def handle_g(subcmd: str) -> Reply: return ''.join(__get_reg_value(*entry) for entry in self.regsmap) - def handle_G(subcmd: str) -> Reply: data = subcmd @@ -268,7 +267,6 @@ def handle_G(subcmd: str) -> Reply: return REPLY_OK - def handle_H(subcmd: str) -> Reply: op = subcmd[0] @@ -277,14 +275,12 @@ def handle_H(subcmd: str) -> Reply: return REPLY_EMPTY - def handle_k(subcmd: str) -> Reply: global killed killed = True return REPLY_OK - def handle_m(subcmd: str) -> Reply: """Read target memory. """ @@ -298,7 +294,6 @@ def handle_m(subcmd: str) -> Reply: else: return data - def handle_M(subcmd: str) -> Reply: """Write target memory. """ @@ -318,7 +313,6 @@ def handle_M(subcmd: str) -> Reply: else: return REPLY_OK - def handle_p(subcmd: str) -> Reply: """Read register value by index. """ @@ -327,7 +321,6 @@ def handle_p(subcmd: str) -> Reply: return __get_reg_value(*self.regsmap[idx]) - def handle_P(subcmd: str) -> Reply: """Write register value by index. """ @@ -342,7 +335,6 @@ def handle_P(subcmd: str) -> Reply: return 'E00' - def handle_Q(subcmd: str) -> Reply: """General queries. @@ -365,14 +357,12 @@ def handle_Q(subcmd: str) -> Reply: return REPLY_OK if feature in supported else REPLY_EMPTY - def handle_D(subcmd: str) -> Reply: """Detach. """ return REPLY_OK - def handle_q(subcmd: str) -> Reply: query, *data = subcmd.split(':') @@ -490,7 +480,7 @@ def handle_q(subcmd: str) -> Reply: elif feature == 'auxv' and op == 'read': try: with self.ql.os.fs_mapper.open('/proc/self/auxv', 'rb') as infile: - infile.seek(offset, 0) # SEEK_SET + infile.seek(offset, 0) # SEEK_SET auxv_data = infile.read(length) except FileNotFoundError: @@ -569,7 +559,6 @@ def handle_q(subcmd: str) -> Reply: return REPLY_EMPTY - def handle_v(subcmd: str) -> Reply: if subcmd == 'MustReplyEmpty': return REPLY_EMPTY @@ -636,10 +625,10 @@ def handle_v(subcmd: str) -> Reply: fd = int(fd, 16) os.close(fd) - + if fd in self.fake_procfs: del self.fake_procfs[fd] - + return 'F0' return REPLY_EMPTY @@ -673,7 +662,6 @@ def handle_v(subcmd: str) -> Reply: return REPLY_EMPTY - def handle_s(subcmd: str) -> Reply: """Perform a single step. """ @@ -688,7 +676,6 @@ def handle_s(subcmd: str) -> Reply: return f'S{SIGTRAP:02x}' - def handle_X(subcmd: str) -> Reply: """Write data to memory. """ @@ -707,7 +694,6 @@ def handle_X(subcmd: str) -> Reply: else: return REPLY_OK - def handle_Z(subcmd: str) -> Reply: """Insert breakpoints or watchpoints. """ @@ -728,7 +714,6 @@ def handle_Z(subcmd: str) -> Reply: return REPLY_EMPTY - def handle_z(subcmd: str) -> Reply: """Remove breakpoints or watchpoints. """ @@ -745,7 +730,6 @@ def handle_z(subcmd: str) -> Reply: return REPLY_EMPTY - handlers = { '!': handle_exclaim, '?': handle_qmark, diff --git a/qiling/os/linux/linux.py b/qiling/os/linux/linux.py index ee9a2b06b..cb84abffb 100644 --- a/qiling/os/linux/linux.py +++ b/qiling/os/linux/linux.py @@ -22,6 +22,7 @@ from . import futex from . import thread + class QlOsLinux(QlOsPosix): type = QL_OS.LINUX @@ -50,7 +51,6 @@ def __init__(self, ql: Qiling): self.elf_mem_start = 0x0 self.load() - def load(self): self.futexm = futex.QlLinuxFutexManagement() @@ -99,7 +99,7 @@ def load(self): self.ql.hook_insn(self.hook_syscall, UC_X86_INS_SYSCALL) # Keep test for _cc #self.ql.hook_insn(hook_posix_api, UC_X86_INS_SYSCALL) - self.thread_class = thread.QlLinuxX8664Thread + self.thread_class = thread.QlLinuxX8664Thread elif self.ql.arch.type == QL_ARCH.RISCV: self.ql.arch.enable_float() @@ -121,7 +121,6 @@ def load(self): if getattr(self.fd[i], 'close_on_exec', 0): self.fd[i] = None - def setup_procfs(self): self.fs_mapper.add_fs_mapping(r'/proc/self/auxv', partial(QlProcFS.self_auxv, self)) self.fs_mapper.add_fs_mapping(r'/proc/self/cmdline', partial(QlProcFS.self_cmdline, self)) @@ -132,17 +131,14 @@ def setup_procfs(self): def hook_syscall(self, ql, intno = None): return self.load_syscall() - def register_function_after_load(self, function): if function not in self.function_after_load_list: self.function_after_load_list.append(function) - def run_function_after_load(self): for f in self.function_after_load_list: f() - def run(self): # do not set-up procfs for drivers and shellcode if not self.ql.code and not self.ql.loader.is_driver: @@ -155,14 +151,14 @@ def run(self): if self.ql.code: self.ql.emu_start(self.entry_point, (self.entry_point + len(self.ql.code)), self.ql.timeout, self.ql.count) else: - if self.ql.multithread == True: + if self.ql.multithread: # start multithreading thread_management = thread.QlLinuxThreadManagement(self.ql) self.ql.os.thread_management = thread_management thread_management.run() else: - if self.ql.entry_point is not None: + if self.ql.entry_point is not None: self.ql.loader.elf_entry = self.ql.entry_point elif self.ql.loader.elf_entry != self.ql.loader.entry_point: diff --git a/qiling/os/linux/procfs.py b/qiling/os/linux/procfs.py index 6166bf684..2cc9af6b7 100644 --- a/qiling/os/linux/procfs.py +++ b/qiling/os/linux/procfs.py @@ -1,6 +1,6 @@ import io -from typing import TYPE_CHECKING, AnyStr, Optional, Sized +from typing import TYPE_CHECKING, AnyStr from qiling.os.mapper import QlFsMappedObject @@ -27,9 +27,8 @@ def self_auxv(os: 'QlOsLinux') -> QlFsMappedObject: auxv_data.extend(os.ql.mem.read(auxv_addr, nbytes)) auxv_addr += nbytes - - return io.BytesIO(bytes(auxv_data)) + return io.BytesIO(bytes(auxv_data)) @staticmethod def self_cmdline(os: 'QlOsLinux') -> QlFsMappedObject: @@ -38,7 +37,6 @@ def self_cmdline(os: 'QlOsLinux') -> QlFsMappedObject: return io.BytesIO(cmdline) - @staticmethod def self_environ(os: 'QlOsLinux') -> QlFsMappedObject: def __to_bytes(s: AnyStr) -> bytes: @@ -52,20 +50,19 @@ def __to_bytes(s: AnyStr) -> bytes: return io.BytesIO(environ) - @staticmethod def self_exe(os: 'QlOsLinux') -> QlFsMappedObject: with open(os.ql.path, 'rb') as exefile: content = exefile.read() return io.BytesIO(content) - + @staticmethod def self_map(mem: 'QlMemoryManager') -> QlFsMappedObject: - content = b"" + content = bytearray() mapinfo = mem.get_mapinfo() for lbound, ubound, perms, label, container in mapinfo: content += f"{lbound:x}-{ubound:x}\t{perms}p\t0\t00:00\t0\t{container if container else label}\n".encode("utf-8") - return io.BytesIO(content) + return io.BytesIO(bytes(content)) diff --git a/qiling/os/mapper.py b/qiling/os/mapper.py index 111fb47ce..d6e8b170a 100644 --- a/qiling/os/mapper.py +++ b/qiling/os/mapper.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # @@ -21,34 +21,34 @@ class QlFsMappedObject: def __init__(self): pass - + def read(self, expected_len): raise NotImplementedError("QlFsMappedObject method not implemented: read") - + def write(self, buffer): raise NotImplementedError("QlFsMappedObject method not implemented: write") - + def fileno(self): raise NotImplementedError("QlFsMappedObject method not implemented: fileno") - + def lseek(self, lseek_offset, lseek_origin): raise NotImplementedError("QlFsMappedObject method not implemented: lseek") - + def close(self): raise NotImplementedError("QlFsMappedObject method not implemented: close") - + def fstat(self): raise NotImplementedError("QlFsMappedObject method not implemented: fstat") - + def ioctl(self, ioctl_cmd, ioctl_arg): raise NotImplementedError("QlFsMappedObject method not implemented: ioctl") def tell(self): raise NotImplementedError("QlFsMappedObject method not implemented: tell") - + def dup(self): raise NotImplementedError("QlFsMappedObject method not implemented: dup") - + def readline(self, end = b'\n'): raise NotImplementedError("QlFsMappedObject method not implemented: readline") @@ -146,7 +146,7 @@ def add_fs_mapping(self, ql_path: Union[os.PathLike, str], real_dest: Union[str, real_dest = self._parse_path(real_dest) self._mapping[ql_path] = real_dest - + def remove_fs_mapping(self, ql_path: Union[os.PathLike, str]): """Remove a mapping from the fs mapper.