From c363dab2fb5cf3487ccc256f69ed219dd6b28707 Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 14 May 2025 13:04:15 +0300 Subject: [PATCH 1/5] Tidy up coverage classes and methods --- qiling/extensions/coverage/formats/base.py | 19 ++++--- qiling/extensions/coverage/formats/drcov.py | 49 ++++++++++--------- .../coverage/formats/drcov_exact.py | 7 +-- qiling/extensions/coverage/formats/ezcov.py | 35 +++++++------ qiling/extensions/tracing/formats/base.py | 18 +++---- 5 files changed, 64 insertions(+), 64 deletions(-) diff --git a/qiling/extensions/coverage/formats/base.py b/qiling/extensions/coverage/formats/base.py index d9fe7c34e..32a6fc550 100644 --- a/qiling/extensions/coverage/formats/base.py +++ b/qiling/extensions/coverage/formats/base.py @@ -4,8 +4,11 @@ # from abc import ABC, abstractmethod +from typing import TYPE_CHECKING -from qiling import Qiling + +if TYPE_CHECKING: + from qiling import Qiling class QlBaseCoverage(ABC): @@ -15,25 +18,21 @@ class QlBaseCoverage(ABC): all the methods marked with the @abstractmethod decorator. """ + FORMAT_NAME: str + def __init__(self, ql: Qiling): super().__init__() self.ql = ql - @property - @staticmethod - @abstractmethod - def FORMAT_NAME() -> str: - raise NotImplementedError - @abstractmethod - def activate(self): + def activate(self) -> None: pass @abstractmethod - def deactivate(self): + def deactivate(self) -> None: pass @abstractmethod - def dump_coverage(self, coverage_file: str): + def dump_coverage(self, coverage_file: str) -> None: pass diff --git a/qiling/extensions/coverage/formats/drcov.py b/qiling/extensions/coverage/formats/drcov.py index 51a421946..d541837d3 100644 --- a/qiling/extensions/coverage/formats/drcov.py +++ b/qiling/extensions/coverage/formats/drcov.py @@ -3,12 +3,18 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -from ctypes import Structure -from ctypes import c_uint32, c_uint16 +from __future__ import annotations + +from ctypes import Structure, c_uint32, c_uint16 +from typing import TYPE_CHECKING, BinaryIO from .base import QlBaseCoverage +if TYPE_CHECKING: + from qiling import Qiling + + # Adapted from https://www.ayrx.me/drcov-file-format class bb_entry(Structure): _fields_ = [ @@ -29,7 +35,7 @@ class QlDrCoverage(QlBaseCoverage): FORMAT_NAME = "drcov" - def __init__(self, ql): + def __init__(self, ql: Qiling): super().__init__(ql) self.drcov_version = 2 @@ -37,28 +43,27 @@ def __init__(self, ql): self.basic_blocks = [] self.bb_callback = None - @staticmethod - def block_callback(ql, address, size, self): - for mod_id, mod in enumerate(ql.loader.images): - if mod.base <= address <= mod.end: - ent = bb_entry(address - mod.base, size, mod_id) - self.basic_blocks.append(ent) - break + def activate(self) -> None: + self.bb_callback = self.ql.hook_block(self.block_callback) - def activate(self): - self.bb_callback = self.ql.hook_block(self.block_callback, user_data=self) + def deactivate(self) -> None: + if self.bb_callback: + self.ql.hook_del(self.bb_callback) - def deactivate(self): - self.ql.hook_del(self.bb_callback) + def dump_coverage(self, coverage_file: str) -> None: + def __write_line(bio: BinaryIO, line: str) -> None: + bio.write(f'{line}\n'.encode()) - def dump_coverage(self, coverage_file): with open(coverage_file, "wb") as cov: - cov.write(f"DRCOV VERSION: {self.drcov_version}\n".encode()) - cov.write(f"DRCOV FLAVOR: {self.drcov_flavor}\n".encode()) - cov.write(f"Module Table: version {self.drcov_version}, count {len(self.ql.loader.images)}\n".encode()) - cov.write("Columns: id, base, end, entry, checksum, timestamp, path\n".encode()) + __write_line(cov, f"DRCOV VERSION: {self.drcov_version}") + __write_line(cov, f"DRCOV FLAVOR: {self.drcov_flavor}") + __write_line(cov, f"Module Table: version {self.drcov_version}, count {len(self.ql.loader.images)}") + __write_line(cov, "Columns: id, base, end, entry, checksum, timestamp, path") + for mod_id, mod in enumerate(self. ql.loader.images): - cov.write(f"{mod_id}, {mod.base}, {mod.end}, 0, 0, 0, {mod.path}\n".encode()) - cov.write(f"BB Table: {len(self.basic_blocks)} bbs\n".encode()) - for bb in self.basic_blocks: + __write_line(cov, f"{mod_id}, {mod.base}, {mod.end}, 0, 0, 0, {mod.path}") + + __write_line(cov, f"BB Table: {len(self.basic_blocks)} bbs") + + for bb in self.basic_blocks.values(): cov.write(bytes(bb)) diff --git a/qiling/extensions/coverage/formats/drcov_exact.py b/qiling/extensions/coverage/formats/drcov_exact.py index 685c6c044..afb6e8446 100644 --- a/qiling/extensions/coverage/formats/drcov_exact.py +++ b/qiling/extensions/coverage/formats/drcov_exact.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # @@ -17,10 +17,7 @@ class QlDrCoverageExact(QlDrCoverage): FORMAT_NAME = "drcov_exact" - def __init__(self, ql): - super().__init__(ql) - - def activate(self): + def activate(self) -> None: # We treat every instruction as a block on its own. self.bb_callback = self.ql.hook_code(self.block_callback, user_data=self) \ No newline at end of file diff --git a/qiling/extensions/coverage/formats/ezcov.py b/qiling/extensions/coverage/formats/ezcov.py index b25218691..2fab67e96 100644 --- a/qiling/extensions/coverage/formats/ezcov.py +++ b/qiling/extensions/coverage/formats/ezcov.py @@ -1,14 +1,20 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +from __future__ import annotations from collections import namedtuple from os.path import basename +from typing import TYPE_CHECKING, List from .base import QlBaseCoverage +if TYPE_CHECKING: + from qiling import Qiling + + # Adapted from https://github.com/nccgroup/Cartographer/blob/main/EZCOV.md#coverage-data class bb_entry(namedtuple('bb_entry', 'offset size mod_id')): def csvline(self): @@ -27,27 +33,24 @@ class QlEzCoverage(QlBaseCoverage): FORMAT_NAME = "ezcov" - def __init__(self, ql): + def __init__(self, ql: Qiling): super().__init__(ql) + self.ezcov_version = 1 - self.ezcov_flavor = 'ezcov' - self.basic_blocks = [] - self.bb_callback = None + self.ezcov_flavor = 'ezcov' + self.basic_blocks: List[bb_entry] = [] + self.bb_callback = None - @staticmethod - def block_callback(ql, address, size, self): - mod = ql.loader.find_containing_image(address) - if mod is not None: - ent = bb_entry(address - mod.base, size, basename(mod.path)) - self.basic_blocks.append(ent) + def block_callback(self, ql: Qiling, address: int, size: int): - def activate(self): - self.bb_callback = self.ql.hook_block(self.block_callback, user_data=self) + def activate(self) -> None: + self.bb_callback = self.ql.hook_block(self.block_callback) - def deactivate(self): - self.ql.hook_del(self.bb_callback) + def deactivate(self) -> None: + if self.bb_callback: + self.ql.hook_del(self.bb_callback) - def dump_coverage(self, coverage_file): + def dump_coverage(self, coverage_file: str) -> None: with open(coverage_file, "w") as cov: cov.write(f"EZCOV VERSION: {self.ezcov_version}\n") cov.write("# Qiling EZCOV exporter tool\n") diff --git a/qiling/extensions/tracing/formats/base.py b/qiling/extensions/tracing/formats/base.py index 145944340..dbb9c78af 100644 --- a/qiling/extensions/tracing/formats/base.py +++ b/qiling/extensions/tracing/formats/base.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # This code structure is copied and modified from the coverage extension @@ -12,24 +12,20 @@ class QlBaseTrace(ABC): To add support for a new coverage format, just derive from this class and implement all the methods marked with the @abstractmethod decorator. """ - + + FORMAT_NAME: str + def __init__(self): super().__init__() - @property - @staticmethod - @abstractmethod - def FORMAT_NAME(): - raise NotImplementedError - @abstractmethod - def activate(self): + def activate(self) -> None: pass @abstractmethod - def deactivate(self): + def deactivate(self) -> None: pass @abstractmethod - def dump_trace(self, trace_file): + def dump_trace(self, trace_file: str) -> None: pass \ No newline at end of file From be824ad0d4671347c6750331c54212933f6aaf1c Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 14 May 2025 13:05:53 +0300 Subject: [PATCH 2/5] Remove drcove bb dups and speed it up --- qiling/extensions/coverage/formats/drcov.py | 32 +++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/qiling/extensions/coverage/formats/drcov.py b/qiling/extensions/coverage/formats/drcov.py index d541837d3..bed0f8701 100644 --- a/qiling/extensions/coverage/formats/drcov.py +++ b/qiling/extensions/coverage/formats/drcov.py @@ -6,13 +6,15 @@ from __future__ import annotations from ctypes import Structure, c_uint32, c_uint16 -from typing import TYPE_CHECKING, BinaryIO +from functools import lru_cache +from typing import TYPE_CHECKING, BinaryIO, Dict, Tuple from .base import QlBaseCoverage if TYPE_CHECKING: from qiling import Qiling + from qiling.loader.loader import QlLoader # Adapted from https://www.ayrx.me/drcov-file-format @@ -40,9 +42,35 @@ def __init__(self, ql: Qiling): self.drcov_version = 2 self.drcov_flavor = 'drcov' - self.basic_blocks = [] + self.basic_blocks: Dict[int, bb_entry] = {} self.bb_callback = None + @lru_cache(maxsize=64) + def _get_img_base(self, loader: QlLoader, address: int) -> Tuple[int, int]: + """Retrieve the containing image of a given address. + + Addresses are expected to be aligned to page boundary, and cached for faster retrieval. + """ + + return next((i, img.base) for i, img in enumerate(loader.images) if img.base <= address < img.end) + + def block_callback(self, ql: Qiling, address: int, size: int): + if address not in self.basic_blocks: + try: + # we rely on the fact that images are allocated on page size boundary and + # use it to speed up image retrieval. we align the basic block address to + # page boundary, knowing basic blocks within the same page belong to the + # same image. then we use the aligned address to retreive the containing + # image. returned values are cached so subsequent retrievals for basic + # blocks within the same page will return the cached value instead of + # going through the retreival process again (up to maxsize cached pages) + + i, img_base = self._get_img_base(ql.loader, address & ~(0x1000 - 1)) + except StopIteration: + pass + else: + self.basic_blocks[address] = bb_entry(address - img_base, size, i) + def activate(self) -> None: self.bb_callback = self.ql.hook_block(self.block_callback) From 104a3f3399c13cdf9f4eb487fcad8d3d5aa8eded Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 14 May 2025 13:06:07 +0300 Subject: [PATCH 3/5] Additional housekeeping --- .../coverage/formats/drcov_exact.py | 3 +-- qiling/extensions/coverage/formats/ezcov.py | 25 +++++++++++++------ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/qiling/extensions/coverage/formats/drcov_exact.py b/qiling/extensions/coverage/formats/drcov_exact.py index afb6e8446..4f7082789 100644 --- a/qiling/extensions/coverage/formats/drcov_exact.py +++ b/qiling/extensions/coverage/formats/drcov_exact.py @@ -19,5 +19,4 @@ class QlDrCoverageExact(QlDrCoverage): def activate(self) -> None: # We treat every instruction as a block on its own. - self.bb_callback = self.ql.hook_code(self.block_callback, user_data=self) - \ No newline at end of file + self.bb_callback = self.ql.hook_code(self.block_callback) diff --git a/qiling/extensions/coverage/formats/ezcov.py b/qiling/extensions/coverage/formats/ezcov.py index 2fab67e96..46290e4c5 100644 --- a/qiling/extensions/coverage/formats/ezcov.py +++ b/qiling/extensions/coverage/formats/ezcov.py @@ -4,9 +4,9 @@ # from __future__ import annotations -from collections import namedtuple -from os.path import basename -from typing import TYPE_CHECKING, List + +import os +from typing import Any, TYPE_CHECKING, List, NamedTuple from .base import QlBaseCoverage @@ -16,10 +16,15 @@ # Adapted from https://github.com/nccgroup/Cartographer/blob/main/EZCOV.md#coverage-data -class bb_entry(namedtuple('bb_entry', 'offset size mod_id')): - def csvline(self): - offset = '0x{:08x}'.format(self.offset) +class bb_entry(NamedTuple): + offset: int + size: int + mod_id: Any + + def as_csv(self) -> str: + offset = f'{self.offset:#010x}' mod_id = f"[ {self.mod_id if self.mod_id is not None else ''} ]" + return f"{offset},{self.size},{mod_id}\n" class QlEzCoverage(QlBaseCoverage): @@ -42,6 +47,10 @@ def __init__(self, ql: Qiling): self.bb_callback = None def block_callback(self, ql: Qiling, address: int, size: int): + img = ql.loader.find_containing_image(address) + + if img is not None: + self.basic_blocks.append(bb_entry(address - img.base, size, os.path.basename(img.path))) def activate(self) -> None: self.bb_callback = self.ql.hook_block(self.block_callback) @@ -54,5 +63,5 @@ def dump_coverage(self, coverage_file: str) -> None: with open(coverage_file, "w") as cov: cov.write(f"EZCOV VERSION: {self.ezcov_version}\n") cov.write("# Qiling EZCOV exporter tool\n") - for bb in self.basic_blocks: - cov.write(bb.csvline()) \ No newline at end of file + + cov.writelines(bb.as_csv() for bb in self.basic_blocks) From 43b8b3c5088bb69dd58b8319142f46039b2a39e9 Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 14 May 2025 13:23:44 +0300 Subject: [PATCH 4/5] Add forgotten import --- qiling/extensions/coverage/formats/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qiling/extensions/coverage/formats/base.py b/qiling/extensions/coverage/formats/base.py index 32a6fc550..4ca162cb8 100644 --- a/qiling/extensions/coverage/formats/base.py +++ b/qiling/extensions/coverage/formats/base.py @@ -3,6 +3,8 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +from __future__ import annotations + from abc import ABC, abstractmethod from typing import TYPE_CHECKING From 70255afd5a5a31ea36b69fd20b832065ab9070fb Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 14 May 2025 13:24:16 +0300 Subject: [PATCH 5/5] Have blob loader contain an image --- qiling/loader/blob.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/qiling/loader/blob.py b/qiling/loader/blob.py index 382dbb33c..b8831a552 100644 --- a/qiling/loader/blob.py +++ b/qiling/loader/blob.py @@ -4,9 +4,10 @@ # from qiling import Qiling -from qiling.loader.loader import QlLoader +from qiling.loader.loader import QlLoader, Image from qiling.os.memory import QlMemoryHeap + class QlLoaderBLOB(QlLoader): def __init__(self, ql: Qiling): super().__init__(ql) @@ -16,13 +17,19 @@ def __init__(self, ql: Qiling): def run(self): self.load_address = self.ql.os.entry_point # for consistency - self.ql.mem.map(self.ql.os.entry_point, self.ql.os.code_ram_size, info="[code]") - self.ql.mem.write(self.ql.os.entry_point, self.ql.code) + code_begins = self.load_address + code_size = self.ql.os.code_ram_size + code_ends = code_begins + code_size - heap_address = self.ql.os.entry_point + self.ql.os.code_ram_size - heap_size = int(self.ql.os.profile.get("CODE", "heap_size"), 16) - self.ql.os.heap = QlMemoryHeap(self.ql, heap_address, heap_address + heap_size) + self.ql.mem.map(code_begins, code_size, info="[code]") + self.ql.mem.write(code_begins, self.ql.code) - self.ql.arch.regs.arch_sp = heap_address - 0x1000 + # allow image-related functionalities + self.images.append(Image(code_begins, code_ends, 'blob_code')) + + # FIXME: heap starts above end of ram?? + heap_base = code_ends + heap_size = int(self.ql.os.profile.get("CODE", "heap_size"), 16) + self.ql.os.heap = QlMemoryHeap(self.ql, heap_base, heap_base + heap_size) - return + self.ql.arch.regs.arch_sp = code_ends - 0x1000