From f7af07907c49ce696a260c387a294857fcd84a09 Mon Sep 17 00:00:00 2001 From: me Date: Fri, 24 Feb 2023 15:03:26 -0500 Subject: [PATCH 1/9] added tests to assert history has length, and to test the case where you want to filter history from multiple shared objects --- tests/test_history.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_history.py b/tests/test_history.py index 6ad2d84d4..1aaf6dba2 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -62,6 +62,17 @@ def test_get_ins_exclude_lib(self): map_for_ins = history.get_mem_map_from_addr(block) self.assertNotRegex(map_for_ins[3], ".*libc.so.*") + assert len(non_libc_blocks) > 0 + + non_libc_blocks_and_ld = history.get_ins_exclude_lib([".*libc.so.*", "ld-linux.*"]) + + for block in non_libc_blocks_and_ld: + map_for_ins = history.get_mem_map_from_addr(block) + self.assertNotRegex(map_for_ins[3], ".*libc.so.*|ld-linux.*") + + assert len(non_libc_blocks_and_ld) > 0 + + def test_get_ins_only_lib(self): ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_hello"], "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.OFF) history = History(ql) @@ -75,5 +86,15 @@ def test_get_ins_only_lib(self): map_for_ins = history.get_mem_map_from_addr(block) self.assertRegex(map_for_ins[3], ".*libc.so.*") + assert len(non_libc_blocks) > 0 + + non_libc_blocks_and_ld = history.get_ins_only_lib([".*libc.so.*", "ld-linux.*"]) + + for block in non_libc_blocks_and_ld : + map_for_ins = history.get_mem_map_from_addr(block) + self.assertRegex(map_for_ins[3], ".*libc.so.*|.*ld-linux.*") + + assert len(non_libc_blocks_and_ld) > 0 + if __name__ == "__main__": unittest.main() \ No newline at end of file From 6da3dbeae216be648a26343458f79fcef5a7beef Mon Sep 17 00:00:00 2001 From: me Date: Fri, 24 Feb 2023 15:04:22 -0500 Subject: [PATCH 2/9] fixed issue where the get_ins_exclude_lib wasnt handling filters for multiple shared objects properly --- qiling/extensions/coverage/formats/history.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/qiling/extensions/coverage/formats/history.py b/qiling/extensions/coverage/formats/history.py index 2edd28e2a..0f1acb3d4 100644 --- a/qiling/extensions/coverage/formats/history.py +++ b/qiling/extensions/coverage/formats/history.py @@ -65,10 +65,11 @@ def get_ins_only_lib(self, libs: List[str]) -> List[int]: List[int]: A list of addresses that have been executed and in the memory maps that match the regex Examples: - >>> history.get_ins_only_lib(["libc.so", "libpthread.so"]) + >>> history.get_ins_only_lib([".*libc.so.*", ".*libpthread.so.*"]) """ executable_maps = self.get_regex_matching_exec_maps(libs) + return [x for x in self.history if any([x >= start and x <= end for start, end, _, _, _ in executable_maps])] def get_ins_exclude_lib(self, libs: list) -> List: @@ -81,11 +82,22 @@ def get_ins_exclude_lib(self, libs: list) -> List: List: A list of addresses that have been executed and are not in the memory maps that match the regex Examples: - >>> history.get_ins_exclude_lib(["libc.so", "libpthread.so"]) + >>> history.get_ins_exclude_lib([".*libc.so.*", ".*libpthread.so.*"]) ''' executable_maps = self.get_regex_matching_exec_maps(libs) - return [x for x in self.history if any([x < start or x > end for start, end, _, _, _ in executable_maps])] + instructions = [] + for h in self.history: + save = True + for start, end, _, _, _ in executable_maps: + if h >= start and h <= end: + save = False + break + + if save: + instructions.append(h) + return instructions + def get_mem_map_from_addr(self, ins: int) -> tuple: '''Returns the memory map that contains the instruction @@ -122,7 +134,7 @@ def get_regex_matching_exec_maps(self, libs: List) -> List: List: A list of tuples that match the regex and are executable Examples: - >>> history.get_regex_matching_exec_maps(["libc.so", "libpthread.so"]) + >>> history.get_regex_matching_exec_maps([".*libc.so.*", ".*libpthread.so.*"]) >>> history.get_regex_matching_exec_maps(".*libc.*") ''' From b7a68d7949f9e4dba5d93617e56e5467fd6933f6 Mon Sep 17 00:00:00 2001 From: me Date: Tue, 28 Feb 2023 13:59:47 -0500 Subject: [PATCH 3/9] brought back the list comprehension (thanks @elicn) and reimplemented history functions to return capstone objects because they are nicer to look at than just integers (addresses) --- qiling/extensions/coverage/formats/history.py | 60 +++++++++++-------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/qiling/extensions/coverage/formats/history.py b/qiling/extensions/coverage/formats/history.py index 0f1acb3d4..ae5200af8 100644 --- a/qiling/extensions/coverage/formats/history.py +++ b/qiling/extensions/coverage/formats/history.py @@ -2,15 +2,18 @@ from qiling import Qiling from qiling.core_hooks_types import HookRet import re +from capstone import CsInsn, Cs class History: history_hook_handle: HookRet = None - history: List[int] = [] + history: List[CsInsn] = [] ql: Qiling + md: Cs def __init__(self, ql: Qiling) -> None: self.ql = ql self.track_block_coverage() + self.md = ql.arch.disassembler def clear_history(self) -> None: """Clears the current state of the history @@ -37,7 +40,13 @@ def track_block_coverage(self) -> None: self.clear_hooks() def __hook_block(ql, address, size): - self.history.append(address) + #0x10 is way more than enough bytes to grab a single instruction + ins_bytes = ql.mem.read(address, 0x10) + try: + self.history.append(next(self.md.disasm(ins_bytes, address))) + except StopIteration: + #if this ever happens, then the unicorn/qiling is going to crash because it tried to execute an instruction that it cant, so we are just not going to do anything + pass self.history_hook_handle = self.ql.hook_block(__hook_block) @@ -49,20 +58,26 @@ def track_instruction_coverage(self) -> None: """ if self.history_hook_handle: self.clear_hooks() - + def __hook_block(ql, address, size): - self.history.append(address) + #0x10 is way more than enough bytes to grab a single instruction + ins_bytes = ql.mem.read(address, 0x10) + try: + self.history.append(next(self.md.disasm(ins_bytes, address))) + except StopIteration: + #if this ever happens, then the unicorn/qiling is going to crash because it tried to execute an instruction that it cant, so we are just not going to do anything + pass self.history_hook_handle = self.ql.hook_code(__hook_block) - def get_ins_only_lib(self, libs: List[str]) -> List[int]: + def get_ins_only_lib(self, libs: List[str]) -> List[CsInsn]: """Returns a list of addresses that have been executed that are only in mmaps for objects that match the regex of items in the list Args: libs (List[str]): A list of regex strings to match against the library names in the memory maps Returns: - List[int]: A list of addresses that have been executed and in the memory maps that match the regex + List[capstone.CsInsn]: A list of CsInsn that have been executed and are only in the memory maps that match the regex Examples: >>> history.get_ins_only_lib([".*libc.so.*", ".*libpthread.so.*"]) @@ -70,40 +85,30 @@ def get_ins_only_lib(self, libs: List[str]) -> List[int]: executable_maps = self.get_regex_matching_exec_maps(libs) - return [x for x in self.history if any([x >= start and x <= end for start, end, _, _, _ in executable_maps])] + return [x for x in self.history if any([x.address >= start and x.address <= end for start, end, _, _, _ in executable_maps])] - def get_ins_exclude_lib(self, libs: list) -> List: + def get_ins_exclude_lib(self, libs: List[str]) -> List[CsInsn]: '''Returns a list of history instructions that are not in the libraries that match the regex in the libs list Args: - libs (List): A list of regex strings to match against the library names in the memory maps + libs (List[str]): A list of regex strings to match against the library names in the memory maps Returns: - List: A list of addresses that have been executed and are not in the memory maps that match the regex + List[capstone.CsInsn]: A list of CsInsn that have been executed and are not in the memory maps that match the regex Examples: >>> history.get_ins_exclude_lib([".*libc.so.*", ".*libpthread.so.*"]) ''' executable_maps = self.get_regex_matching_exec_maps(libs) - instructions = [] - for h in self.history: - save = True - for start, end, _, _, _ in executable_maps: - if h >= start and h <= end: - save = False - break - - if save: - instructions.append(h) - return instructions + return [h for h in self.history if not any(start <= h.address <= end for start, end, _, _, _ in executable_maps)] - def get_mem_map_from_addr(self, ins: int) -> tuple: + def get_mem_map_from_addr(self, ins) -> tuple: '''Returns the memory map that contains the instruction Args: - ins (int): The instruction address to search for + ins: The instruction address to search for, can be either an int or a capstone.CsInsn Returns: tuple: A tuple that contains the memory map that contains the instruction @@ -113,6 +118,11 @@ def get_mem_map_from_addr(self, ins: int) -> tuple: >>> history.get_mem_map_from_addr(0x7ffff7dd1b97) ''' + if isinstance(ins, CsInsn): + ins = ins.address + + assert isinstance(ins, int) + #get the memory map that contains the instruction mem_map = [x for x in self.ql.mem.get_mapinfo() if x[0] <= ins and x[1] >= ins] @@ -122,13 +132,13 @@ def get_mem_map_from_addr(self, ins: int) -> tuple: # i sure hope theres not more than one map that contains the instruction lol return mem_map[0] - def get_regex_matching_exec_maps(self, libs: List) -> List: + def get_regex_matching_exec_maps(self, libs: List[str]) -> List: '''Returns a list of tuples for current mmaps whose names match the regex of libs in the list This is a wrapper around ql.mem.get_mapinfo() and just filters the results by the regex of the library names and also only returns maps that are executable Args: - libs (List): A list of regex strings to match against the library names in the memory maps + libs (List[str]): A list of regex strings to match against the library names in the memory maps Returns: List: A list of tuples that match the regex and are executable From b805e73b94bbb6e3ed068c87971b27732717428d Mon Sep 17 00:00:00 2001 From: me Date: Mon, 6 Mar 2023 10:56:50 -0500 Subject: [PATCH 4/9] made more changes as per the comments in PR 1317 --- qiling/extensions/coverage/formats/history.py | 76 +++++++++---------- 1 file changed, 35 insertions(+), 41 deletions(-) diff --git a/qiling/extensions/coverage/formats/history.py b/qiling/extensions/coverage/formats/history.py index ae5200af8..991b9a882 100644 --- a/qiling/extensions/coverage/formats/history.py +++ b/qiling/extensions/coverage/formats/history.py @@ -1,19 +1,23 @@ -from typing import List -from qiling import Qiling -from qiling.core_hooks_types import HookRet +from typing import List, Tuple, TYPE_CHECKING + +if TYPE_CHECKING: + from qiling import Qiling + from qiling.core_hooks_types import HookRet + from capstone import Cs + import re -from capstone import CsInsn, Cs +from functools import partial +from capstone import CsInsn class History: - history_hook_handle: HookRet = None - history: List[CsInsn] = [] - ql: Qiling - md: Cs + history_hook_handle: 'HookRet' = None + history: 'List[CsInsn]' = [] + ql: 'Qiling' + md: 'Cs' - def __init__(self, ql: Qiling) -> None: + def __init__(self, ql: 'Qiling') -> None: self.ql = ql self.track_block_coverage() - self.md = ql.arch.disassembler def clear_history(self) -> None: """Clears the current state of the history @@ -30,6 +34,17 @@ def clear_hooks(self) -> None: self.ql.hook_del(self.history_hook_handle) + #a python function hook block to be used in both track_block_coverage and track_instruction_coverage + @staticmethod + def __hook_block(history_obj, ql, address, size): + #0x10 is way more than enough bytes to grab a single instruction + ins_bytes = ql.mem.read(address, 0x10) + try: + history_obj.history.append(next(ql.arch.disassembler.disasm(ins_bytes, address))) + except StopIteration: + #if this ever happens, then the unicorn/qiling is going to crash because it tried to execute an instruction that it cant, so we are just not going to do anything + pass + def track_block_coverage(self) -> None: """Configures the history plugin to track all of the basic blocks that are executed. Removes any existing hooks @@ -39,16 +54,9 @@ def track_block_coverage(self) -> None: if self.history_hook_handle: self.clear_hooks() - def __hook_block(ql, address, size): - #0x10 is way more than enough bytes to grab a single instruction - ins_bytes = ql.mem.read(address, 0x10) - try: - self.history.append(next(self.md.disasm(ins_bytes, address))) - except StopIteration: - #if this ever happens, then the unicorn/qiling is going to crash because it tried to execute an instruction that it cant, so we are just not going to do anything - pass + partial_hook_block = partial(self.__hook_block, self) - self.history_hook_handle = self.ql.hook_block(__hook_block) + self.history_hook_handle = self.ql.hook_block(partial_hook_block) def track_instruction_coverage(self) -> None: """Configures the history plugin to track all of the instructions that are executed. Removes any existing hooks @@ -59,18 +67,11 @@ def track_instruction_coverage(self) -> None: if self.history_hook_handle: self.clear_hooks() - def __hook_block(ql, address, size): - #0x10 is way more than enough bytes to grab a single instruction - ins_bytes = ql.mem.read(address, 0x10) - try: - self.history.append(next(self.md.disasm(ins_bytes, address))) - except StopIteration: - #if this ever happens, then the unicorn/qiling is going to crash because it tried to execute an instruction that it cant, so we are just not going to do anything - pass + partial_hook_block = partial(self.__hook_block, self) - self.history_hook_handle = self.ql.hook_code(__hook_block) + self.history_hook_handle = self.ql.hook_code(partial_hook_block) - def get_ins_only_lib(self, libs: List[str]) -> List[CsInsn]: + def get_ins_only_lib(self, libs: 'List[str]') -> 'List[CsInsn]': """Returns a list of addresses that have been executed that are only in mmaps for objects that match the regex of items in the list Args: @@ -85,9 +86,9 @@ def get_ins_only_lib(self, libs: List[str]) -> List[CsInsn]: executable_maps = self.get_regex_matching_exec_maps(libs) - return [x for x in self.history if any([x.address >= start and x.address <= end for start, end, _, _, _ in executable_maps])] + return [x for x in self.history if any(start <= x.address <= end for start, end, _, _, _ in executable_maps)] - def get_ins_exclude_lib(self, libs: List[str]) -> List[CsInsn]: + def get_ins_exclude_lib(self, libs: 'List[str]') -> 'List[CsInsn]': '''Returns a list of history instructions that are not in the libraries that match the regex in the libs list Args: @@ -123,22 +124,15 @@ def get_mem_map_from_addr(self, ins) -> tuple: assert isinstance(ins, int) - #get the memory map that contains the instruction - mem_map = [x for x in self.ql.mem.get_mapinfo() if x[0] <= ins and x[1] >= ins] + return next((x for x in self.ql.mem.get_mapinfo() if x[0] <= ins and x[1] >= ins), None) - if len(mem_map) == 0: - return None - - # i sure hope theres not more than one map that contains the instruction lol - return mem_map[0] - - def get_regex_matching_exec_maps(self, libs: List[str]) -> List: + def get_regex_matching_exec_maps(self, libs: 'List[str]') -> List: '''Returns a list of tuples for current mmaps whose names match the regex of libs in the list This is a wrapper around ql.mem.get_mapinfo() and just filters the results by the regex of the library names and also only returns maps that are executable Args: - libs (List[str]): A list of regex strings to match against the library names in the memory maps + libs (Union[str, Collection[str]]): A list of regex strings to match against the library names in the memory maps Returns: List: A list of tuples that match the regex and are executable From 31f33f1dae42e67c4b2beeb7971b61c3635d2314 Mon Sep 17 00:00:00 2001 From: me Date: Mon, 6 Mar 2023 17:24:37 -0500 Subject: [PATCH 5/9] added the __future__ annotations so i dont have to use quotes for the types, __hook_block is a class method now because it makes sense, --- qiling/extensions/coverage/formats/history.py | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/qiling/extensions/coverage/formats/history.py b/qiling/extensions/coverage/formats/history.py index 991b9a882..4df8c06b3 100644 --- a/qiling/extensions/coverage/formats/history.py +++ b/qiling/extensions/coverage/formats/history.py @@ -1,3 +1,4 @@ +from __future__ import annotations from typing import List, Tuple, TYPE_CHECKING if TYPE_CHECKING: @@ -10,12 +11,12 @@ from capstone import CsInsn class History: - history_hook_handle: 'HookRet' = None - history: 'List[CsInsn]' = [] - ql: 'Qiling' - md: 'Cs' + history_hook_handle: HookRet = None + history: List[CsInsn] = [] + ql: Qiling + md: Cs - def __init__(self, ql: 'Qiling') -> None: + def __init__(self, ql: Qiling) -> None: self.ql = ql self.track_block_coverage() @@ -23,7 +24,7 @@ def clear_history(self) -> None: """Clears the current state of the history """ - self.history = [] + self.history.clear() def clear_hooks(self) -> None: """Clears the current history hook from the Qiling instance @@ -35,12 +36,11 @@ def clear_hooks(self) -> None: self.ql.hook_del(self.history_hook_handle) #a python function hook block to be used in both track_block_coverage and track_instruction_coverage - @staticmethod - def __hook_block(history_obj, ql, address, size): + def __hook_block(self, ql, address, size): #0x10 is way more than enough bytes to grab a single instruction ins_bytes = ql.mem.read(address, 0x10) try: - history_obj.history.append(next(ql.arch.disassembler.disasm(ins_bytes, address))) + self.history.append(next(ql.arch.disassembler.disasm(ins_bytes, address))) except StopIteration: #if this ever happens, then the unicorn/qiling is going to crash because it tried to execute an instruction that it cant, so we are just not going to do anything pass @@ -54,9 +54,7 @@ def track_block_coverage(self) -> None: if self.history_hook_handle: self.clear_hooks() - partial_hook_block = partial(self.__hook_block, self) - - self.history_hook_handle = self.ql.hook_block(partial_hook_block) + self.history_hook_handle = self.ql.hook_block(self.__hook_block) def track_instruction_coverage(self) -> None: """Configures the history plugin to track all of the instructions that are executed. Removes any existing hooks @@ -67,11 +65,9 @@ def track_instruction_coverage(self) -> None: if self.history_hook_handle: self.clear_hooks() - partial_hook_block = partial(self.__hook_block, self) - - self.history_hook_handle = self.ql.hook_code(partial_hook_block) + self.history_hook_handle = self.ql.hook_code(self.__hook_block) - def get_ins_only_lib(self, libs: 'List[str]') -> 'List[CsInsn]': + def get_ins_only_lib(self, libs: List[str]) -> List[CsInsn]: """Returns a list of addresses that have been executed that are only in mmaps for objects that match the regex of items in the list Args: @@ -88,7 +84,7 @@ def get_ins_only_lib(self, libs: 'List[str]') -> 'List[CsInsn]': return [x for x in self.history if any(start <= x.address <= end for start, end, _, _, _ in executable_maps)] - def get_ins_exclude_lib(self, libs: 'List[str]') -> 'List[CsInsn]': + def get_ins_exclude_lib(self, libs: List[str]) -> List[CsInsn]: '''Returns a list of history instructions that are not in the libraries that match the regex in the libs list Args: @@ -105,7 +101,7 @@ def get_ins_exclude_lib(self, libs: 'List[str]') -> 'List[CsInsn]': return [h for h in self.history if not any(start <= h.address <= end for start, end, _, _, _ in executable_maps)] - def get_mem_map_from_addr(self, ins) -> tuple: + def get_mem_map_from_addr(self, ins) -> Tuple: '''Returns the memory map that contains the instruction Args: @@ -126,7 +122,7 @@ def get_mem_map_from_addr(self, ins) -> tuple: return next((x for x in self.ql.mem.get_mapinfo() if x[0] <= ins and x[1] >= ins), None) - def get_regex_matching_exec_maps(self, libs: 'List[str]') -> List: + def get_regex_matching_exec_maps(self, libs: List[str]) -> List: '''Returns a list of tuples for current mmaps whose names match the regex of libs in the list This is a wrapper around ql.mem.get_mapinfo() and just filters the results by the regex of the library names and also only returns maps that are executable From 1fb316be132739f45b1e50e1a35d551dcb1fe61a Mon Sep 17 00:00:00 2001 From: me Date: Mon, 6 Mar 2023 18:12:05 -0500 Subject: [PATCH 6/9] added some more type annotations, added a new callback specifically for dealing with arm arch, and not accessing the ql.arch.disassember directly anymore --- qiling/extensions/coverage/formats/history.py | 53 ++++++++++++++++--- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/qiling/extensions/coverage/formats/history.py b/qiling/extensions/coverage/formats/history.py index 4df8c06b3..bef38f1e5 100644 --- a/qiling/extensions/coverage/formats/history.py +++ b/qiling/extensions/coverage/formats/history.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import List, Tuple, TYPE_CHECKING +from typing import List, Tuple, TYPE_CHECKING, Union if TYPE_CHECKING: from qiling import Qiling @@ -9,15 +9,19 @@ import re from functools import partial from capstone import CsInsn +from qiling.const import QL_ARCH class History: history_hook_handle: HookRet = None history: List[CsInsn] = [] ql: Qiling md: Cs + arm_is_thumb: bool def __init__(self, ql: Qiling) -> None: self.ql = ql + self.md = self.ql.arch.disassembler + self.arm_is_thumb = getattr(ql.arch, 'is_thumb', False) self.track_block_coverage() def clear_history(self) -> None: @@ -36,11 +40,36 @@ def clear_hooks(self) -> None: self.ql.hook_del(self.history_hook_handle) #a python function hook block to be used in both track_block_coverage and track_instruction_coverage - def __hook_block(self, ql, address, size): + def __hook_block(self, ql: Qiling, address: int, size: int): + ''' + The unicorn block/instruction hook function for the track_block_coverage and track_instruction_coverage functions. This just give us a way to append capstone objects to the history list + + NOTE: this is not supposed to be used in arm architecture, use __hook_block_arm instead (if you use this, things are going to be hella slow) + ''' + #0x10 is way more than enough bytes to grab a single instruction + ins_bytes = ql.mem.read(address, 0x10) + try: + self.history.append(next(self.md.disasm(ins_bytes, address))) + except StopIteration: + #if this ever happens, then the unicorn/qiling is going to crash because it tried to execute an instruction that it cant, so we are just not going to do anything + pass + + # in arm we have to worry about thumb mode, so we have our own hook function + def __hook_block_arm(self, ql: Qiling, address: int, size: int): + ''' + The unicorn block/instruction hook function for the track_block_coverage and track_instruction_coverage functions. This just give us a way to append capstone objects to the history list + ''' + + #get the current state of the thumb mode + if self.arm_is_thumb != ql.arch.is_thumb: + #the thumb mode has changed, so we need to update the disassembler + self.arm_is_thumb = ql.arch.is_thumb + self.md = self.ql.arch.disassembler + #0x10 is way more than enough bytes to grab a single instruction ins_bytes = ql.mem.read(address, 0x10) try: - self.history.append(next(ql.arch.disassembler.disasm(ins_bytes, address))) + self.history.append(next(self.md.disasm(ins_bytes, address))) except StopIteration: #if this ever happens, then the unicorn/qiling is going to crash because it tried to execute an instruction that it cant, so we are just not going to do anything pass @@ -53,8 +82,12 @@ def track_block_coverage(self) -> None: """ if self.history_hook_handle: self.clear_hooks() - - self.history_hook_handle = self.ql.hook_block(self.__hook_block) + + hook = self.__hook_block + if self.ql.arch.type == QL_ARCH.ARM: + hook = self.__hook_block_arm + + self.history_hook_handle = self.ql.hook_block(hook) def track_instruction_coverage(self) -> None: """Configures the history plugin to track all of the instructions that are executed. Removes any existing hooks @@ -65,7 +98,11 @@ def track_instruction_coverage(self) -> None: if self.history_hook_handle: self.clear_hooks() - self.history_hook_handle = self.ql.hook_code(self.__hook_block) + hook = self.__hook_block + if self.ql.arch.type == QL_ARCH.ARM: + hook = self.__hook_block_arm + + self.history_hook_handle = self.ql.hook_code(hook) def get_ins_only_lib(self, libs: List[str]) -> List[CsInsn]: """Returns a list of addresses that have been executed that are only in mmaps for objects that match the regex of items in the list @@ -101,11 +138,11 @@ def get_ins_exclude_lib(self, libs: List[str]) -> List[CsInsn]: return [h for h in self.history if not any(start <= h.address <= end for start, end, _, _, _ in executable_maps)] - def get_mem_map_from_addr(self, ins) -> Tuple: + def get_mem_map_from_addr(self, ins: Union[int, CsInsn]) -> Tuple: '''Returns the memory map that contains the instruction Args: - ins: The instruction address to search for, can be either an int or a capstone.CsInsn + ins (Union[int, CsInsn]): The instruction address to search for, can be either an int or a capstone.CsInsn Returns: tuple: A tuple that contains the memory map that contains the instruction From 6f0d6beec4d82857bc7006c61329b6c18dc3f7bb Mon Sep 17 00:00:00 2001 From: me Date: Mon, 6 Mar 2023 19:05:19 -0500 Subject: [PATCH 7/9] cleaned up some pylint stuff, consolidated the arm and other arch block callback --- qiling/extensions/coverage/formats/history.py | 77 +++++++------------ 1 file changed, 26 insertions(+), 51 deletions(-) diff --git a/qiling/extensions/coverage/formats/history.py b/qiling/extensions/coverage/formats/history.py index bef38f1e5..cd1731da2 100644 --- a/qiling/extensions/coverage/formats/history.py +++ b/qiling/extensions/coverage/formats/history.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import List, Tuple, TYPE_CHECKING, Union +from typing import List, Tuple, TYPE_CHECKING, Union, Optional, Any if TYPE_CHECKING: from qiling import Qiling @@ -7,12 +7,11 @@ from capstone import Cs import re -from functools import partial from capstone import CsInsn -from qiling.const import QL_ARCH + class History: - history_hook_handle: HookRet = None + history_hook_handle: HookRet history: List[CsInsn] = [] ql: Qiling md: Cs @@ -39,39 +38,25 @@ def clear_hooks(self) -> None: self.ql.hook_del(self.history_hook_handle) - #a python function hook block to be used in both track_block_coverage and track_instruction_coverage - def __hook_block(self, ql: Qiling, address: int, size: int): + def __hook_block(self, ql: Qiling, address: int, size: int, *context: Any) -> Any: ''' The unicorn block/instruction hook function for the track_block_coverage and track_instruction_coverage functions. This just give us a way to append capstone objects to the history list - - NOTE: this is not supposed to be used in arm architecture, use __hook_block_arm instead (if you use this, things are going to be hella slow) ''' - #0x10 is way more than enough bytes to grab a single instruction - ins_bytes = ql.mem.read(address, 0x10) - try: - self.history.append(next(self.md.disasm(ins_bytes, address))) - except StopIteration: - #if this ever happens, then the unicorn/qiling is going to crash because it tried to execute an instruction that it cant, so we are just not going to do anything - pass - # in arm we have to worry about thumb mode, so we have our own hook function - def __hook_block_arm(self, ql: Qiling, address: int, size: int): - ''' - The unicorn block/instruction hook function for the track_block_coverage and track_instruction_coverage functions. This just give us a way to append capstone objects to the history list - ''' - - #get the current state of the thumb mode - if self.arm_is_thumb != ql.arch.is_thumb: - #the thumb mode has changed, so we need to update the disassembler - self.arm_is_thumb = ql.arch.is_thumb + # get the current state of the thumb mode + + if self.arm_is_thumb is not getattr(ql.arch, "is_thumb", False): + # the thumb mode has changed, so we need to update the disassembler + self.arm_is_thumb = not self.arm_is_thumb self.md = self.ql.arch.disassembler - #0x10 is way more than enough bytes to grab a single instruction + # 0x10 is way more than enough bytes to grab a single instruction ins_bytes = ql.mem.read(address, 0x10) - try: + try: self.history.append(next(self.md.disasm(ins_bytes, address))) - except StopIteration: - #if this ever happens, then the unicorn/qiling is going to crash because it tried to execute an instruction that it cant, so we are just not going to do anything + except StopIteration: + # if this ever happens, then the unicorn/qiling is going to crash because it tried to execute + # an instruction that it cant, so we are just not going to do anything pass def track_block_coverage(self) -> None: @@ -80,14 +65,10 @@ def track_block_coverage(self) -> None: Returns: None """ - if self.history_hook_handle: + if getattr(self, 'history_hook_handle', None): self.clear_hooks() - hook = self.__hook_block - if self.ql.arch.type == QL_ARCH.ARM: - hook = self.__hook_block_arm - - self.history_hook_handle = self.ql.hook_block(hook) + self.history_hook_handle = self.ql.hook_block(self.__hook_block) def track_instruction_coverage(self) -> None: """Configures the history plugin to track all of the instructions that are executed. Removes any existing hooks @@ -95,14 +76,10 @@ def track_instruction_coverage(self) -> None: Returns: None """ - if self.history_hook_handle: + if getattr(self, 'history_hook_handle', None): self.clear_hooks() - hook = self.__hook_block - if self.ql.arch.type == QL_ARCH.ARM: - hook = self.__hook_block_arm - - self.history_hook_handle = self.ql.hook_code(hook) + self.history_hook_handle = self.ql.hook_code(self.__hook_block) def get_ins_only_lib(self, libs: List[str]) -> List[CsInsn]: """Returns a list of addresses that have been executed that are only in mmaps for objects that match the regex of items in the list @@ -137,15 +114,14 @@ def get_ins_exclude_lib(self, libs: List[str]) -> List[CsInsn]: executable_maps = self.get_regex_matching_exec_maps(libs) return [h for h in self.history if not any(start <= h.address <= end for start, end, _, _, _ in executable_maps)] - - def get_mem_map_from_addr(self, ins: Union[int, CsInsn]) -> Tuple: + def get_mem_map_from_addr(self, ins: Union[int, CsInsn]) -> Optional[Tuple]: '''Returns the memory map that contains the instruction Args: ins (Union[int, CsInsn]): The instruction address to search for, can be either an int or a capstone.CsInsn Returns: - tuple: A tuple that contains the memory map that contains the instruction + Optional[Tuple]: A tuple that contains the memory map that contains the instruction this tuple is in the format of (start_addr, end_addr, perms, name, path) Examples: @@ -159,16 +135,17 @@ def get_mem_map_from_addr(self, ins: Union[int, CsInsn]) -> Tuple: return next((x for x in self.ql.mem.get_mapinfo() if x[0] <= ins and x[1] >= ins), None) - def get_regex_matching_exec_maps(self, libs: List[str]) -> List: + def get_regex_matching_exec_maps(self, libs: List[str]) -> List[Tuple]: '''Returns a list of tuples for current mmaps whose names match the regex of libs in the list - This is a wrapper around ql.mem.get_mapinfo() and just filters the results by the regex of the library names and also only returns maps that are executable + This is a wrapper around ql.mem.get_mapinfo() and just filters the results by the regex of the library names + and also only returns maps that are executable Args: libs (Union[str, Collection[str]]): A list of regex strings to match against the library names in the memory maps Returns: - List: A list of tuples that match the regex and are executable + List[Tuple]: A list of tuples that match the regex and are executable Examples: >>> history.get_regex_matching_exec_maps([".*libc.so.*", ".*libpthread.so.*"]) @@ -184,9 +161,7 @@ def get_regex_matching_exec_maps(self, libs: List[str]) -> List: # filter the list of tuples # so that we return only the ones where the library name matches the regex - regex_matching_libs = [x for x in self.ql.mem.get_mapinfo() if any([r.match(x[3]) for r in regex])] + regex_matching_libs = [x for x in self.ql.mem.get_mapinfo() if any(r.match(x[3]) for r in regex)] # filter viable_libs for items that have the executable bit set - executable_maps = [x for x in regex_matching_libs if 'x' in x[2]] - - return executable_maps \ No newline at end of file + return [x for x in regex_matching_libs if 'x' in x[2]] From 11b61e0e913984e4e587270bbfed9fb5c5ff353b Mon Sep 17 00:00:00 2001 From: me Date: Mon, 6 Mar 2023 19:17:32 -0500 Subject: [PATCH 8/9] commenting on why we have this arm handler in the __hook_block --- qiling/extensions/coverage/formats/history.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qiling/extensions/coverage/formats/history.py b/qiling/extensions/coverage/formats/history.py index cd1731da2..2afc1960f 100644 --- a/qiling/extensions/coverage/formats/history.py +++ b/qiling/extensions/coverage/formats/history.py @@ -43,8 +43,9 @@ def __hook_block(self, ql: Qiling, address: int, size: int, *context: Any) -> An The unicorn block/instruction hook function for the track_block_coverage and track_instruction_coverage functions. This just give us a way to append capstone objects to the history list ''' - # get the current state of the thumb mode - + # get the current state of the thumb mode, only applys to arm + # originally we were going to access the ql.arch.disassembler directly for all architectures from in this callback, but in the + # implementation for arch.arm.disassembler, the capstone instance is recreated every time (to make sure THUMB mode is properly dealt with) if self.arm_is_thumb is not getattr(ql.arch, "is_thumb", False): # the thumb mode has changed, so we need to update the disassembler self.arm_is_thumb = not self.arm_is_thumb From c3ba489c0eb51e197c5f148f7f30b26a630decf9 Mon Sep 17 00:00:00 2001 From: me Date: Mon, 6 Mar 2023 19:24:40 -0500 Subject: [PATCH 9/9] removing "*context" argument from history tracker callback --- qiling/extensions/coverage/formats/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiling/extensions/coverage/formats/history.py b/qiling/extensions/coverage/formats/history.py index 2afc1960f..b0415b41b 100644 --- a/qiling/extensions/coverage/formats/history.py +++ b/qiling/extensions/coverage/formats/history.py @@ -38,7 +38,7 @@ def clear_hooks(self) -> None: self.ql.hook_del(self.history_hook_handle) - def __hook_block(self, ql: Qiling, address: int, size: int, *context: Any) -> Any: + def __hook_block(self, ql: Qiling, address: int, size: int) -> Any: ''' The unicorn block/instruction hook function for the track_block_coverage and track_instruction_coverage functions. This just give us a way to append capstone objects to the history list '''