Maintenance PR

examples/scripts/dllscollector.bat

@@ -57,6 +57,7 @@ CALL :collect_dll32 kdcom.dll
 CALL :collect_dll32 kernel32.dll
 CALL :collect_dll32 KernelBase.dll
 CALL :collect_dll32 mpr.dll
+CALL :collect_dll32 mscoree.dll
 CALL :collect_dll32 msvcp_win.dll
 CALL :collect_dll32 msvcp60.dll
 CALL :collect_dll32 msvcr120_clr0400.dll, msvcr110.dll
@@ -102,6 +103,7 @@ CALL :collect_dll64 advapi32.dll
 CALL :collect_dll64 gdi32.dll
 CALL :collect_dll64 kernel32.dll
 CALL :collect_dll64 KernelBase.dll
+CALL :collect_dll64 mscoree.dll
 CALL :collect_dll64 msvcrt.dll
 CALL :collect_dll64 ntdll.dll
 CALL :collect_dll64 ntoskrnl.exe

qiling/arch/x86_utils.py

@@ -1,5 +1,6 @@
 
 from abc import abstractmethod
+from typing import Optional
 
 from qiling import Qiling
 from qiling.arch.x86 import QlArchIntel
@@ -32,7 +33,7 @@ def __setitem__(self, index: int, data: bytes) -> None:
 
         self.mem.write(self.base + (index * self.entsize), data)
 
-    def get_next_free(self, start: int = None, end: int = None) -> int:
+    def get_next_free(self, start: Optional[int] = None, end: Optional[int] = None) -> int:
         # The Linux kernel determines whether the segment is empty by judging whether the content in the current GDT segment is 0.
         null_entry = b'\x00' * self.entsize
 
@@ -112,7 +113,7 @@ def get_entry(self, index: int) -> bytes:
     def set_entry(self, index: int, data: bytes) -> None:
         self.array[index] = data
 
-    def get_free_idx(self, start: int = None, end: int = None) -> int:
+    def get_free_idx(self, start: Optional[int] = None, end: Optional[int] = None) -> int:
         return self.array.get_next_free(start, end)
 
 

qiling/debugger/gdb/gdb.py

@@ -378,7 +378,7 @@ def handle_q(subcmd: str) -> Reply:
 
             if query == 'Supported':
                 # list of supported features excluding the multithreading-related ones
-                common = (
+                features = [
                     'BreakpointCommands+',
                     'ConditionalBreakpoints+',
                     'ConditionalTracepoints+',
@@ -408,23 +408,21 @@ def handle_q(subcmd: str) -> Reply:
                     'multiprocess+',
                     'no-resumed+',
                     'qXfer:auxv:read+',
-                    'qXfer:exec-file:read+',
                     'qXfer:features:read+',
                     # 'qXfer:libraries-svr4:read+',
                     # 'qXfer:osdata:read+',
                     'qXfer:siginfo:read+',
                     'qXfer:siginfo:write+',
                     'qXfer:statictrace:read+',
-                    'qXfer:threads:read+',
                     'qXfer:traceframe-info:read+',
                     'swbreak+',
                     'tracenz+',
                     'vfork-events+'
-                )
+                ]
 
                 # might or might not need for multi thread
                 if self.ql.multithread:
-                    features = (
+                    features += [
                         'PacketSize=47ff',
                         'FastTracepoints+',
                         'QThreadEvents+',
@@ -436,16 +434,26 @@ def handle_q(subcmd: str) -> Reply:
                         'qXfer:btrace-conf:read+',
                         'qXfer:btrace:read+',
                         'vContSupported+'
-                    )
+                    ]
 
                 else:
-                    features = (
+                    features += [
                         'PacketSize=3fff',
                         'qXfer:spu:read+',
                         'qXfer:spu:write+'
-                    )
+                    ]
+
+                # os dependent features
+                if not self.ql.interpreter:
+                    # filesystem dependent features
+                    if hasattr(self.ql.os, 'path'):
+                        features.append('qXfer:exec-file:read+')
 
-                return ';'.join(common + features)
+                    # process dependent features
+                    if hasattr(self.ql.os, 'pid'):
+                        features.append('qXfer:threads:read+')
+
+                return ';'.join(features)
 
             elif query == 'Xfer':
                 feature, op, annex, params = data
@@ -462,15 +470,11 @@ def handle_q(subcmd: str) -> Reply:
                     return f'{"l" if len(content) < length else "m"}{content}'
 
                 elif feature == 'threads' and op == 'read':
-                    if not self.ql.baremetal and hasattr(self.ql.os, 'pid'):
-                        content = '\r\n'.join((
-                            '<threads>',
-                            f'<thread id="{self.ql.os.pid}" core="1" name="{self.ql.targetname}"/>',
-                            '</threads>'
-                        ))
-
-                    else:
-                        content = ''
+                    content = '\r\n'.join((
+                        '<threads>',
+                        f'<thread id="{self.ql.os.pid}" core="1" name="{self.ql.targetname}"/>',
+                        '</threads>'
+                    ))
 
                     return f'l{content}'
 
@@ -494,7 +498,7 @@ def handle_q(subcmd: str) -> Reply:
                     return b'l' + auxv_data[:length]
 
                 elif feature == 'exec-file' and op == 'read':
-                    return f'l{os.path.abspath(self.ql.path)}'
+                    return f'l{self.ql.os.path.host_to_virtual_path(self.ql.path)}'
 
                 elif feature == 'libraries-svr4' and op == 'read':
                     # TODO: this one requires information of loaded libraries which currently not provided
@@ -584,16 +588,20 @@ def handle_v(subcmd: str) -> Reply:
                         flags = int(flags, 16)
                         mode = int(mode, 16)
 
-                        # try to guess whether this is an emulated path or real one
-                        if path.startswith(os.path.abspath(self.ql.rootfs)):
-                            host_path = path
+                        virtpath = self.ql.os.path.virtual_abspath(path)
+
+                        if virtpath.startswith(r'/proc'):
+                            # TODO: we really need a centralized virtual filesystem to open
+                            # both emulated (like procfs) and real files, and manage their
+                            # file descriptors seamlessly
+                            fd = -1
                         else:
                             host_path = self.ql.os.path.virtual_to_host_path(path)
 
-                        self.ql.log.debug(f'{PROMPT} target file: {host_path}')
+                            self.ql.log.debug(f'{PROMPT} target host path: {host_path}')
 
-                        if os.path.exists(host_path) and not path.startswith(r'/proc'):
-                            fd = os.open(host_path, flags, mode)
+                            if os.path.exists(host_path):
+                                fd = os.open(host_path, flags, mode)
 
                     return f'F{fd:x}'
 

qiling/extensions/trace.py

@@ -97,7 +97,7 @@ def __parse_op(op: X86Op) -> str:
 		"""
 
 		if op.type == CS_OP_REG:
-			return insn.reg_name(op.value.reg)
+			return insn.reg_name(op.value.reg) or '?'
 
 		elif op.type == CS_OP_IMM:
 			imm = op.value.imm
@@ -113,6 +113,7 @@ def __parse_op(op: X86Op) -> str:
 			disp  = mem.disp
 
 			ea = base + index * scale + disp
+			seg = f'{insn.reg_name(mem.segment)}:' if mem.segment else ''
 
 			# we construct the string representation for each operand; denote memory
 			# dereferenes with the appropriate 'ptr' prefix. the 'lea' instruction is
@@ -131,7 +132,7 @@ def __parse_op(op: X86Op) -> str:
 
 				qualifier = f'{ptr} ptr '
 
-			return f'{qualifier}[{__resolve(ea) or f"{ea:#x}"}]'
+			return f'{qualifier}{seg}[{__resolve(ea) or f"{ea:#x}"}]'
 
 		# unexpected op type
 		raise RuntimeError

qiling/loader/pe.py

@@ -248,7 +248,12 @@ def load_dll(self, name: str, is_driver: bool = False) -> int:
             # in case of a dll loaded from a hooked API call, failures would not be
             # recoverable and we have to give up its DllMain.
             if not self.ql.os.PE_RUN:
+
+                # temporarily set PE_RUN to allow proper fcall unwinding during
+                # execution of DllMain
+                self.ql.os.PE_RUN = True
                 self.call_dll_entrypoint(dll, dll_base, dll_len, dll_name)
+                self.ql.os.PE_RUN = False
 
         self.ql.log.info(f'Done loading {dll_name}')
 
@@ -622,7 +627,9 @@ def init_ki_user_shared_data(self):
         user_shared_data_obj = KUSER_SHARED_DATA()
         user_shared_data_size = ctypes.sizeof(KUSER_SHARED_DATA)
 
-        self.ql.mem.map(addr, self.ql.mem.align_up(user_shared_data_size))
+        # TODO: initialize key fields in this structure
+
+        self.ql.mem.map(addr, self.ql.mem.align_up(user_shared_data_size), info='[kuser shared]')
         self.ql.mem.write(addr, bytes(user_shared_data_obj))
 
     def init_security_cookie(self, pe: pefile.PE, image_base: int):
@@ -657,6 +664,7 @@ def run(self):
         self.sys_dlls = (
             'ntdll.dll',
             'kernel32.dll',
+            'mscoree.dll',
             'ucrtbase.dll'
         )
 
@@ -695,7 +703,6 @@ def run(self):
         cmdline = ntpath.join(self.ql.os.userprofile, 'Desktop', self.ql.targetname)
         cmdargs = ' '.join(f'"{arg}"' if ' ' in arg else arg for arg in self.argv[1:])
 
-        self.filepath = bytes(f'{cmdline}\x00', "utf-8")
         self.cmdline = bytes(f'{cmdline} {cmdargs}\x00', "utf-8")
 
         self.load(pe)
@@ -722,14 +729,13 @@ def load(self, pe: Optional[pefile.PE]):
             self.ql.log.info(f'Loading {self.path} to {image_base:#x}')
             self.ql.log.info(f'PE entry point at {self.entry_point:#x}')
 
-            self.ql.mem.map(image_base, image_size, info=f'[{image_name}]')
+            self.ql.mem.map(image_base, image_size, info=f'{image_name}')
             self.images.append(Image(image_base, image_base + pe.NT_HEADERS.OPTIONAL_HEADER.SizeOfImage, os.path.abspath(self.path)))
 
             if self.is_driver:
                 self.init_driver_object()
                 self.init_registry_path()
                 self.init_eprocess()
-                self.init_ki_user_shared_data()
 
                 # set IRQ Level in CR8 to PASSIVE_LEVEL
                 self.ql.arch.regs.write(UC_X86_REG_CR8, 0)
@@ -748,6 +754,8 @@ def load(self, pe: Optional[pefile.PE]):
                 # add image to ldr table
                 self.add_ldr_data_table_entry(image_name)
 
+            self.init_ki_user_shared_data()
+
             pe.parse_data_directories()
 
             # done manipulating pe file; write its contents into memory
@@ -817,8 +825,6 @@ def load(self, pe: Optional[pefile.PE]):
                 self.ql.os.fcall.call_native(self.entry_point, args, None)
 
         elif pe is None:
-            self.filepath = b""
-
             self.ql.mem.map(self.entry_point, self.ql.os.code_ram_size, info="[shellcode]")
 
             self.init_teb()

qiling/os/fcall.py

@@ -2,7 +2,7 @@
 # 
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
 
-from typing import Any, Callable, Iterable, MutableMapping, Optional, Mapping, Tuple, Sequence
+from typing import Any, Callable, Iterable, Iterator, MutableMapping, Optional, Mapping, Tuple, Sequence
 
 from qiling import Qiling
 from qiling.cc import QlCC
@@ -50,6 +50,21 @@ def __make_accessor(nbits: int) -> Accessor:
 		# let the user override default accessors or add custom ones
 		self.accessors.update(accessors)
 
+	def readEllipsis(self, ptypes: Sequence[Any]) -> Iterator[int]:
+		"""
+		"""
+
+		default = self.accessors[PARAM_INTN]
+
+		# count skipped slots
+		si = sum(self.accessors.get(typ, default)[2] for typ in ptypes)
+
+		while True:
+			read, _, nslots = default
+
+			yield read(si)
+			si += nslots
+
 	def readParams(self, ptypes: Sequence[Any]) -> Sequence[int]:
 		"""Walk the function parameters list and get their values.
 

qiling/os/linux/kernel_api/kernel_api.py

@@ -31,12 +31,10 @@ def hook_printk(ql: Qiling, address: int, params):
         return 0
 
     level = int(format[1]) if format[1].isdigit() else 4
-    nargs = format.count("%")
-    ptypes = (POINTER, ) + (PARAM_INTN, ) * nargs
-    args = ql.os.fcall.readParams(ptypes)[1:]
+    args = ql.os.fcall.readEllipsis(params.values())
 
-    count = ql.os.utils.printf(f'{PRINTK_LEVEL[level]} {format[2:]}', args, wstring=False)
-    ql.os.utils.update_ellipsis(params, args)
+    count, upd_args = ql.os.utils.printf(f'{PRINTK_LEVEL[level]} {format[2:]}', args, wstring=False)
+    upd_args(params)
 
     return count
 

qiling/os/linux/thread.py

@@ -11,6 +11,7 @@
 from unicorn.unicorn import UcError
 
 from qiling import Qiling
+from qiling.const import QL_ARCH
 from qiling.os.thread import *
 from qiling.arch.x86_const import *
 from qiling.exception import QlErrorExecutionStop

qiling/os/mapper.py

@@ -101,15 +101,17 @@ def open_ql_file(self, path: str, openflags: int, openmode: int):
         if self.has_mapping(path):
             return self._open_mapping_ql_file(path, openflags, openmode)
 
-        real_path = self.path.transform_to_real_path(path)
-        return ql_file.open(real_path, openflags, openmode)
+        host_path = self.path.virtual_to_host_path(path)
+
+        return ql_file.open(host_path, openflags, openmode)
 
     def open(self, path: str, openmode: str):
         if self.has_mapping(path):
             return self._open_mapping(path, openmode)
 
-        real_path = self.path.transform_to_real_path(path)
-        return open(real_path, openmode)
+        host_path = self.path.virtual_to_host_path(path)
+
+        return open(host_path, openmode)
 
     def _parse_path(self, p: Union[os.PathLike, str]) -> str:
         fspath = getattr(p, '__fspath__', None)

qiling/os/path.py

@@ -42,9 +42,9 @@ def __init__(self, rootfs: str, cwd: str, emulos: QL_OS) -> None:
         self.cwd = cwd
 
         # <TEMPORARY>
+        # temporary aliases for backward compatibility
         self.transform_to_relative_path = self.virtual_abspath
         self.transform_to_real_path = self.virtual_to_host_path
-        self.transform_to_link_path = self.virtual_to_host_path
         # </TEMPORARY>
 
     @staticmethod
@@ -258,6 +258,23 @@ def __is_safe_host_path(self, hostpath: Path, strict: bool = False) -> bool:
         else:
             return True
 
+    def host_to_virtual_path(self, hostpath: str) -> str:
+        """Convert a host path to its corresponding virtual path relative to rootfs.
+
+        Args:
+            hostpath : host path
+
+        Returns: the corresponding virtual path
+
+        Raises: `ValueError` in case the host path does not map to a rootfs location
+        """
+
+        resolved = Path(hostpath).resolve(strict=False)
+        virtpath = self._cwd_anchor / resolved.relative_to(self._rootfs_path)
+
+        return str(virtpath)
+
+
     def virtual_abspath(self, virtpath: str) -> str:
         """Convert a relative virtual path to an absolute virtual path based
         on the current working directory.