From 1f3eb48b004dae188d97c4bed1a01191f5036cf4 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 12 Jan 2024 08:44:48 +0200 Subject: [PATCH 1/4] Fix #1436 --- qiling/os/windows/const.py | 7 ++++ qiling/os/windows/dlls/kernel32/heapapi.py | 39 ++++++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/qiling/os/windows/const.py b/qiling/os/windows/const.py index 7168edbe3..7d1134bce 100644 --- a/qiling/os/windows/const.py +++ b/qiling/os/windows/const.py @@ -786,3 +786,10 @@ TIME_ZONE_ID_UNKNOWN = 0 TIME_ZONE_ID_STANDARD = 1 TIME_ZONE_ID_DAYLIGHT = 2 + +# heap management flags +HEAP_NO_SERIALIZE = 0x00000001 +HEAP_GENERATE_EXCEPTIONS = 0x00000004 +HEAP_ZERO_MEMORY = 0x00000008 +HEAP_REALLOC_IN_PLACE_ONLY = 0x00000010 +HEAP_CREATE_ENABLE_EXECUTE = 0x00040000 diff --git a/qiling/os/windows/dlls/kernel32/heapapi.py b/qiling/os/windows/dlls/kernel32/heapapi.py index 43c62e4d8..c48e5aa8f 100644 --- a/qiling/os/windows/dlls/kernel32/heapapi.py +++ b/qiling/os/windows/dlls/kernel32/heapapi.py @@ -3,9 +3,36 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -from qiling import Qiling +from __future__ import annotations +from typing import TYPE_CHECKING + from qiling.os.windows.api import * -from qiling.os.windows.fncc import * +from qiling.os.windows.const import HEAP_ZERO_MEMORY +from qiling.os.windows.fncc import STDCALL, winsdkapi + + +if TYPE_CHECKING: + from qiling import Qiling + from qiling.os.memory import QlMemoryManager + + +def __zero_mem(mem: QlMemoryManager, ptr: int, size: int) -> None: + """Zero a memory range, but avoid hogging to much on host resources. + """ + + # go by page granularity + npages, remainder = divmod(size, mem.pagesize) + + if npages: + zeros = b'\x00' * mem.pagesize + + for _ in range(npages): + mem.write(ptr, zeros) + ptr += len(zeros) + + if remainder: + mem.write(ptr, b'\x00' * remainder) + # HANDLE HeapCreate( # DWORD flOptions, @@ -33,9 +60,15 @@ def hook_HeapCreate(ql: Qiling, address: int, params): 'dwBytes' : SIZE_T }) def hook_HeapAlloc(ql: Qiling, address: int, params): + dwFlags = params["dwFlags"] dwBytes = params["dwBytes"] - return ql.os.heap.alloc(dwBytes) + ptr = ql.os.heap.alloc(dwBytes) + + if ptr and (dwFlags & HEAP_ZERO_MEMORY): + __zero_mem(ql.mem, ptr, dwBytes) + + return ptr # SIZE_T HeapSize( # HANDLE hHeap, From 0e7927015dbe0ae73520422a72a03d7ebbcbb6a4 Mon Sep 17 00:00:00 2001 From: elicn Date: Sat, 13 Jan 2024 19:45:36 +0200 Subject: [PATCH 2/4] Fix #1430 --- qiling/os/windows/dlls/kernel32/fileapi.py | 179 ++++++++++++++++--- qiling/os/windows/dlls/kernel32/handleapi.py | 18 +- 2 files changed, 170 insertions(+), 27 deletions(-) diff --git a/qiling/os/windows/dlls/kernel32/fileapi.py b/qiling/os/windows/dlls/kernel32/fileapi.py index 60465ff87..5efcbfe84 100644 --- a/qiling/os/windows/dlls/kernel32/fileapi.py +++ b/qiling/os/windows/dlls/kernel32/fileapi.py @@ -3,18 +3,20 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +from contextlib import contextmanager import ntpath import os from shutil import copyfile from datetime import datetime +from typing import IO, Optional from qiling import Qiling from qiling.exception import QlErrorNotImplemented from qiling.os.windows.api import * from qiling.os.windows.const import * from qiling.os.windows.fncc import * -from qiling.os.windows.handle import Handle +from qiling.os.windows.handle import Handle, HandleManager from qiling.os.windows.structs import FILETIME, make_win32_find_data # DWORD GetFileType( @@ -604,14 +606,126 @@ def hook_GetFileSize(ql: Qiling, address: int, params): ql.os.last_error = ERROR_INVALID_HANDLE return -1 # INVALID_FILE_SIZE + +class FileMapping: + def read(self, offset: int, size: int) -> bytes: ... + def write(self, offset: int, data: bytes) -> None: ... + + +# TODO: needs to be implemened +class FileMappingMem(FileMapping): + ... + + +class FileMappingFile(FileMapping): + def __init__(self, fobj: IO) -> None: + self._fobj = fobj + + self._read_hook = None + self._write_hook = None + + def map_view(self, ql: Qiling, fbase: int, lbound: int, ubound: int) -> None: + def __read_mapview(ql: Qiling, access: int, addr: int, size: int, _) -> None: + """Fetch the corresponding file part into memory. + """ + + data = self.read(fbase + (addr - lbound), size) + + # FIXME: that triggers the write hook, and may be problematic for read-only ranges + ql.mem.write(addr, data) + + def __write_mapview(ql: Qiling, access: int, addr: int, size: int, value: int) -> None: + """Write data back to the corresponding file part. + """ + + pack = { + 1: ql.pack8, + 2: ql.pack16, + 4: ql.pack32, + 8: ql.pack64 + }[size] + + self.write(fbase + (addr - lbound), pack(value)) + + self._read_hook = ql.hook_mem_read(__read_mapview, begin=lbound, end=ubound) + self._write_hook = ql.hook_mem_write(__write_mapview, begin=lbound, end=ubound) + + def unmap_view(self) -> None: + if self._read_hook: + self._read_hook.remove() + + if self._write_hook: + self._write_hook.remove() + + @contextmanager + def __seek_temporary(self, offset: Optional[int] = None): + """A context manager construct for performing actions that would normaly affect the file + position, but without actually affecting it. + """ + + fpos = self._fobj.tell() + + if offset is not None: + self._fobj.seek(offset) + + try: + yield self._fobj + finally: + self._fobj.seek(fpos) + + def get_file_size(self) -> int: + with self.__seek_temporary() as fobj: + return fobj.seek(0, os.SEEK_END) + + def inc_file_size(self, addendum: int) -> None: + with self.__seek_temporary() as fobj: + fobj.seek(0, os.SEEK_END) + fobj.write(b'\x00' * addendum) + + def read(self, offset: int, size: int) -> bytes: + with self.__seek_temporary(offset) as fobj: + return fobj.read(size) + + def write(self, offset: int, data: bytes) -> None: + with self.__seek_temporary(offset) as fobj: + fobj.write(data) + + def _CreateFileMapping(ql: Qiling, address: int, params): hFile = params['hFile'] + dwMaximumSizeHigh = params['dwMaximumSizeHigh'] + dwMaximumSizeLow = params['dwMaximumSizeLow'] lpName = params['lpName'] - new_handle = Handle(obj=hFile, name=lpName) - ql.os.handle_manager.append(new_handle) + req_size = (dwMaximumSizeHigh << 32) | dwMaximumSizeLow - return new_handle.id + if hFile == INVALID_HANDLE_VALUE: + # TODO: create file mapping backed by memory instead of a file + fmobj = FileMappingMem() + + else: + # look for an existing mapping handle with the same name + if lpName: + existing_handle = ql.os.handle_manager.search(lpName) + + # if found, return it + if existing_handle is not None: + return existing_handle + + fhandle = ql.os.handle_manager.get(hFile) + + # wrap the opened file with an accessor class + fmobj = FileMappingFile(fhandle.obj) + fsize = fmobj.get_file_size() + + # if requeted mapping is size is larger than the file size, enlarge it + if req_size > fsize: + fmobj.inc_file_size(req_size - fsize) + + fm_handle = Handle(obj=fmobj, name=lpName or None) + ql.os.handle_manager.append(fm_handle) + + return fm_handle.id # HANDLE CreateFileMappingA( # HANDLE hFile, @@ -667,29 +781,44 @@ def hook_CreateFileMappingW(ql: Qiling, address: int, params): }) def hook_MapViewOfFile(ql: Qiling, address: int, params): hFileMappingObject = params['hFileMappingObject'] + dwFileOffsetHigh = params['dwFileOffsetHigh'] dwFileOffsetLow = params['dwFileOffsetLow'] dwNumberOfBytesToMap = params['dwNumberOfBytesToMap'] - map_file_handle = ql.os.handle_manager.search_by_obj(hFileMappingObject) + handles: HandleManager = ql.os.handle_manager + fm_handle = handles.get(hFileMappingObject) + + if fm_handle is None: + return 0 + + fmobj = fm_handle.obj + + # the respective file mapping hFile was set to INVALID_HANDLE_VALUE (that is, mapping is backed by page file) + if isinstance(fmobj, FileMappingMem): + raise QlErrorNotImplemented('files mapping backed by page file are not supported yet') - if map_file_handle is None: - ret = ql.os.heap.alloc(dwNumberOfBytesToMap) - new_handle = Handle(obj=hFileMappingObject, name=ret) - ql.os.handle_manager.append(new_handle) else: - ret = map_file_handle.name + offset = (dwFileOffsetHigh << 32) | dwFileOffsetLow + mapview_size = dwNumberOfBytesToMap or (fmobj.get_file_size() - offset) + + if mapview_size < 1: + return 0 - hFile = ql.os.handle_manager.get(hFileMappingObject).obj + mapview = ql.os.heap.alloc(mapview_size) - if ql.os.handle_manager.get(hFile): - f = ql.os.handle_manager.get(hFile).obj + if not mapview: + return 0 - if type(f) is file: - f.seek(dwFileOffsetLow, 0) - data = f.read(dwNumberOfBytesToMap) - ql.mem.write(ret, data) + # read content from file but retain original position + data = fmobj.read(offset, mapview_size) + ql.mem.write(mapview, data) - return ret + fmobj.map_view(ql, offset, mapview, mapview + mapview_size - 1) + + # although file views are not strictly handles, it would be easier to manage them as such + handles.append(Handle(id=mapview, obj=fmobj)) + + return mapview # BOOL UnmapViewOfFile( # LPCVOID lpBaseAddress @@ -700,15 +829,17 @@ def hook_MapViewOfFile(ql: Qiling, address: int, params): def hook_UnmapViewOfFile(ql: Qiling, address: int, params): lpBaseAddress = params['lpBaseAddress'] - map_file_hande = ql.os.handle_manager.search(lpBaseAddress) + handles: HandleManager = ql.os.handle_manager + fv_handle = handles.get(lpBaseAddress) - if not map_file_hande: - return 0 + if fv_handle: + fv_handle.obj.unmap_view() + ql.os.heap.free(lpBaseAddress) + handles.delete(fv_handle.id) - ql.os.heap.free(map_file_hande.name) - ql.os.handle_manager.delete(map_file_hande.id) + return 1 - return 1 + return 0 # BOOL CopyFileA( # LPCSTR lpExistingFileName, diff --git a/qiling/os/windows/dlls/kernel32/handleapi.py b/qiling/os/windows/dlls/kernel32/handleapi.py index bbc4debec..8d1968148 100644 --- a/qiling/os/windows/dlls/kernel32/handleapi.py +++ b/qiling/os/windows/dlls/kernel32/handleapi.py @@ -3,10 +3,18 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -from qiling import Qiling +from __future__ import annotations + +from typing import TYPE_CHECKING, IO + from qiling.os.windows.api import * -from qiling.os.windows.const import * -from qiling.os.windows.fncc import * +from qiling.os.windows.const import ERROR_INVALID_HANDLE, HANDLE_FLAG_PROTECT_FROM_CLOSE +from qiling.os.windows.fncc import STDCALL, winsdkapi + + +if TYPE_CHECKING: + from qiling import Qiling + # BOOL DuplicateHandle( # HANDLE hSourceProcessHandle, @@ -53,6 +61,10 @@ def hook_CloseHandle(ql: Qiling, address: int, params): # FIXME: add error return 0 + # if this a file handle, close it + if isinstance(handle.obj, IO): + handle.obj.close() + ql.os.handle_manager.delete(value) return 1 From effe4f484367a67a679f9db50c5f93a9139e6e3e Mon Sep 17 00:00:00 2001 From: elicn Date: Sat, 13 Jan 2024 20:18:39 +0200 Subject: [PATCH 3/4] Fix #1433 --- qiling/os/utils.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/qiling/os/utils.py b/qiling/os/utils.py index 552f38222..a9241ee6f 100644 --- a/qiling/os/utils.py +++ b/qiling/os/utils.py @@ -153,7 +153,10 @@ def __dup(iterator: Iterator[T], out: List[T]) -> Iterator[T]: va_list = __dup(va_args, orig_args) - read_string = self.read_wstring if wstring else self.read_cstring + read_str = { + False: self.read_cstring, + True: self.read_wstring + } def __repl(m: re.Match) -> str: """Convert printf format string tokens into Python's. @@ -187,9 +190,13 @@ def __repl(m: re.Match) -> str: typ = m['type'] arg = next(va_list) - if typ in 'sS': + if typ == 's': typ = 's' - arg = read_string(arg) + arg = read_str[wstring](arg) + + elif typ == 'S': + typ = 's' + arg = read_str[not wstring](arg) elif typ == 'Z': # note: ANSI_STRING and UNICODE_STRING have identical layout @@ -197,7 +204,7 @@ def __repl(m: re.Match) -> str: with ucstr_struct.ref(self.ql.mem, arg) as ucstr_obj: typ = 's' - arg = read_string(ucstr_obj.Buffer) + arg = read_str[wstring](ucstr_obj.Buffer) elif typ == 'p': pound = '#' From e80fa267378f9dc2c8cb78a8026c63681bd89e6d Mon Sep 17 00:00:00 2001 From: elicn Date: Sat, 13 Jan 2024 21:19:04 +0200 Subject: [PATCH 4/4] Improve file mapping support --- qiling/os/windows/dlls/kernel32/fileapi.py | 23 +++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/qiling/os/windows/dlls/kernel32/fileapi.py b/qiling/os/windows/dlls/kernel32/fileapi.py index 5efcbfe84..fea8f23fb 100644 --- a/qiling/os/windows/dlls/kernel32/fileapi.py +++ b/qiling/os/windows/dlls/kernel32/fileapi.py @@ -608,13 +608,12 @@ def hook_GetFileSize(ql: Qiling, address: int, params): class FileMapping: - def read(self, offset: int, size: int) -> bytes: ... - def write(self, offset: int, data: bytes) -> None: ... + pass -# TODO: needs to be implemened class FileMappingMem(FileMapping): - ... + # mapping backed my page file, for which we simply use memory. no need to do anything really + pass class FileMappingFile(FileMapping): @@ -699,8 +698,7 @@ def _CreateFileMapping(ql: Qiling, address: int, params): req_size = (dwMaximumSizeHigh << 32) | dwMaximumSizeLow - if hFile == INVALID_HANDLE_VALUE: - # TODO: create file mapping backed by memory instead of a file + if hFile == ql.unpack(ql.packs(INVALID_HANDLE_VALUE)): fmobj = FileMappingMem() else: @@ -795,7 +793,10 @@ def hook_MapViewOfFile(ql: Qiling, address: int, params): # the respective file mapping hFile was set to INVALID_HANDLE_VALUE (that is, mapping is backed by page file) if isinstance(fmobj, FileMappingMem): - raise QlErrorNotImplemented('files mapping backed by page file are not supported yet') + mapview = ql.os.heap.alloc(dwNumberOfBytesToMap) + + if not mapview: + return 0 else: offset = (dwFileOffsetHigh << 32) | dwFileOffsetLow @@ -809,7 +810,9 @@ def hook_MapViewOfFile(ql: Qiling, address: int, params): if not mapview: return 0 - # read content from file but retain original position + # read content from file but retain original position. + # not sure this is actually required since all accesses to this memory area are monitored + # and relect file content rather than what is currently in memory data = fmobj.read(offset, mapview_size) ql.mem.write(mapview, data) @@ -833,7 +836,9 @@ def hook_UnmapViewOfFile(ql: Qiling, address: int, params): fv_handle = handles.get(lpBaseAddress) if fv_handle: - fv_handle.obj.unmap_view() + if isinstance(fv_handle.obj, FileMappingFile): + fv_handle.obj.unmap_view() + ql.os.heap.free(lpBaseAddress) handles.delete(fv_handle.id)