From 29762d4e04a763a455eba719f440452efbb4c603 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Fri, 9 Feb 2024 00:52:36 -0500 Subject: [PATCH 01/25] Use python3.8 in docker images --- build_image.py | 2 +- ofrak_core/Dockerstub | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build_image.py b/build_image.py index a2b021e02..cb2226259 100644 --- a/build_image.py +++ b/build_image.py @@ -195,7 +195,7 @@ def create_dockerfile_base(config: OfrakImageConfig) -> str: dockerfile_base_parts += [f"### {dockerstage_path}", dockerstub] dockerfile_base_parts += [ - "FROM python:3.7-bullseye@sha256:338ead05c1a0aa8bd8fcba8e4dbbe2afd0283b4732fd30cf9b3bfcfcbc4affab", + "FROM python:3.8-bullseye@sha256:e1cd369204123e89646f8c001db830eddfe3e381bd5c837df00141be3bd754cb", "", ] diff --git a/ofrak_core/Dockerstub b/ofrak_core/Dockerstub index f4725a6be..b81261935 100644 --- a/ofrak_core/Dockerstub +++ b/ofrak_core/Dockerstub @@ -61,7 +61,7 @@ RUN cd /tmp && \ # Install Jefferson WORKDIR /tmp -RUN wget https://bootstrap.pypa.io/pip/get-pip.py && python3.9 get-pip.py && python3.7 get-pip.py && rm get-pip.py +RUN wget https://bootstrap.pypa.io/pip/get-pip.py && python3.9 get-pip.py && python3.8 get-pip.py && rm get-pip.py RUN python3.9 -m pip install jefferson WORKDIR / From 67ee05ea9cecd411c6d4d24a01c6e98f5cc2632c Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Fri, 9 Feb 2024 01:06:07 -0500 Subject: [PATCH 02/25] Require pytest<8.0 This is needed becase of https://github.com/pytest-dev/pytest/issues/11890 https://github.com/TvoroG/pytest-lazy-fixture/issues/65 --- ofrak_type/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_type/setup.py b/ofrak_type/setup.py index c74f530b4..54e5a162a 100644 --- a/ofrak_type/setup.py +++ b/ofrak_type/setup.py @@ -32,7 +32,7 @@ def run(self): "fun-coverage==0.2.0", "hypothesis~=6.39.3", "mypy==0.942", - "pytest", + "pytest<8.0", "pytest-cov", ] }, From 500ee9b1c6140848a6ff102bf4a6180a77b85d8e Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Fri, 9 Feb 2024 10:33:35 -0500 Subject: [PATCH 03/25] Update changelog --- ofrak_patch_maker/CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ofrak_patch_maker/CHANGELOG.md b/ofrak_patch_maker/CHANGELOG.md index 17f7a66c7..ee82a1330 100644 --- a/ofrak_patch_maker/CHANGELOG.md +++ b/ofrak_patch_maker/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to `ofrak-patch-maker` will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) +### Changed +- Update docker configuration to use Python 3.8, rather than 3.7 ([416](https://github.com/redballoonsecurity/ofrak/pull/416)) + ### Added - GNU 32-bit x86 toolchain. ([#405](https://github.com/redballoonsecurity/ofrak/pull/405)) From 0fe707e7b7227b38937cc90edeac8edf6ba152c7 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Fri, 9 Feb 2024 10:36:17 -0500 Subject: [PATCH 04/25] Revert "Update changelog" This reverts commit 500ee9b1c6140848a6ff102bf4a6180a77b85d8e. Making changes before having coffee :( --- ofrak_patch_maker/CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/ofrak_patch_maker/CHANGELOG.md b/ofrak_patch_maker/CHANGELOG.md index ee82a1330..17f7a66c7 100644 --- a/ofrak_patch_maker/CHANGELOG.md +++ b/ofrak_patch_maker/CHANGELOG.md @@ -4,9 +4,6 @@ All notable changes to `ofrak-patch-maker` will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) -### Changed -- Update docker configuration to use Python 3.8, rather than 3.7 ([416](https://github.com/redballoonsecurity/ofrak/pull/416)) - ### Added - GNU 32-bit x86 toolchain. ([#405](https://github.com/redballoonsecurity/ofrak/pull/405)) From e2a0110dd4d6c3f9971693c31111b8778ae45277 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Fri, 9 Feb 2024 16:28:37 -0500 Subject: [PATCH 05/25] Update to latest angr==9.2.89, which also necessitates Python >= 3.8 and capstone==5.0.0.post1 --- disassemblers/ofrak_angr/CHANGELOG.md | 3 +++ .../ofrak_angr/ofrak_angr/components/blocks/unpackers.py | 2 +- disassemblers/ofrak_angr/requirements.txt | 2 +- disassemblers/ofrak_angr/setup.py | 2 +- disassemblers/ofrak_capstone/CHANGELOG.md | 3 +++ disassemblers/ofrak_capstone/requirements.txt | 2 +- 6 files changed, 10 insertions(+), 4 deletions(-) diff --git a/disassemblers/ofrak_angr/CHANGELOG.md b/disassemblers/ofrak_angr/CHANGELOG.md index eb673296c..b36905321 100644 --- a/disassemblers/ofrak_angr/CHANGELOG.md +++ b/disassemblers/ofrak_angr/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) +### Changed +- Update to latest angr==9.2.89, which also necessitates Python >= 3.8. + ### Fixed - Add `importlib-resources` dependency as workaround for z3-solver dependency issue. ([#401](https://github.com/redballoonsecurity/ofrak/pull/401)) diff --git a/disassemblers/ofrak_angr/ofrak_angr/components/blocks/unpackers.py b/disassemblers/ofrak_angr/ofrak_angr/components/blocks/unpackers.py index e9c34d525..15b7c803b 100644 --- a/disassemblers/ofrak_angr/ofrak_angr/components/blocks/unpackers.py +++ b/disassemblers/ofrak_angr/ofrak_angr/components/blocks/unpackers.py @@ -228,7 +228,7 @@ def _angr_get_dword_blocks( if xref is None or not any(xref in bb_range for bb_range in valid_data_xref_ranges): continue - LOGGER.debug(f"Creating DataWord for {cb_data_xref.content} @ {cb_data_xref_addr:#x}") + LOGGER.debug(f"Creating DataWord for {cb_data_xref.content!r} @ {cb_data_xref_addr:#x}") format_string = endian_flag + dword_size_map[word_size] diff --git a/disassemblers/ofrak_angr/requirements.txt b/disassemblers/ofrak_angr/requirements.txt index a529c3fff..ca01977d1 100644 --- a/disassemblers/ofrak_angr/requirements.txt +++ b/disassemblers/ofrak_angr/requirements.txt @@ -1,2 +1,2 @@ -angr==9.2.6 +angr==9.2.89 importlib-resources # A workaround for https://github.com/redballoonsecurity/ofrak/issues/398 diff --git a/disassemblers/ofrak_angr/setup.py b/disassemblers/ofrak_angr/setup.py index a106800d8..481de0e1b 100644 --- a/disassemblers/ofrak_angr/setup.py +++ b/disassemblers/ofrak_angr/setup.py @@ -69,7 +69,7 @@ def read_requirements(requirements_path): "Topic :: Security", "Typing :: Typed", ], - python_requires=">=3.7", + python_requires=">=3.8", license="Proprietary", license_files=["LICENSE"], cmdclass={"egg_info": egg_info_ex}, diff --git a/disassemblers/ofrak_capstone/CHANGELOG.md b/disassemblers/ofrak_capstone/CHANGELOG.md index 669a71b1c..bc0b6a026 100644 --- a/disassemblers/ofrak_capstone/CHANGELOG.md +++ b/disassemblers/ofrak_capstone/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) +### Changed +- Update to captione==5.0.0.post1. + ## 1.0.0 - 2022-01-25 ### Added Initial release. Hello world! diff --git a/disassemblers/ofrak_capstone/requirements.txt b/disassemblers/ofrak_capstone/requirements.txt index ae1c92132..5903f94cb 100644 --- a/disassemblers/ofrak_capstone/requirements.txt +++ b/disassemblers/ofrak_capstone/requirements.txt @@ -1 +1 @@ -capstone==4.0.2 +capstone==5.0.0.post1 From 24956a393f6321b2bcf358d642feb3b7374c9818 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Fri, 9 Feb 2024 21:13:32 -0500 Subject: [PATCH 06/25] Apply Edward's attempted fix to angr test failure --- disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py b/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py index 4f09974e3..9b27d59ff 100755 --- a/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py +++ b/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py @@ -54,7 +54,7 @@ async def expected_results(self, unpack_verify_test_case: ComplexBlockUnpackerTe 0x110, 0x110, 0x130, - keep_same_is_exit_point=True, + keep_same_is_exit_point=False, ) return self._fixup_test_case_for_pie( From 99f26d56eddf94965043649480a0180530248e9c Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sat, 10 Feb 2024 23:06:57 -0500 Subject: [PATCH 07/25] Add a note on recommending Python 3.8 --- docs/environment-setup.md | 2 +- docs/getting-started.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/environment-setup.md b/docs/environment-setup.md index 7a00c452f..bb4a66aec 100644 --- a/docs/environment-setup.md +++ b/docs/environment-setup.md @@ -1,7 +1,7 @@ # Environment Setup & Installing OFRAK !!! warning - OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! + OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. There are three main ways one can set up an environment to use OFRAK: diff --git a/docs/getting-started.md b/docs/getting-started.md index 7660125b4..8716d56c2 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -3,7 +3,7 @@ ## Quick Start - Unpack a firmware file and display it in the GUI !!! warning - OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! + OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. ```bash pip install ofrak From 5e47d440482577a03f7bde66a3622f6050100c71 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sat, 10 Feb 2024 23:06:57 -0500 Subject: [PATCH 08/25] Add a note on recommending Python 3.8 --- docs/environment-setup.md | 2 +- docs/getting-started.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/environment-setup.md b/docs/environment-setup.md index 7a00c452f..bb4a66aec 100644 --- a/docs/environment-setup.md +++ b/docs/environment-setup.md @@ -1,7 +1,7 @@ # Environment Setup & Installing OFRAK !!! warning - OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! + OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. There are three main ways one can set up an environment to use OFRAK: diff --git a/docs/getting-started.md b/docs/getting-started.md index 7660125b4..8716d56c2 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -3,7 +3,7 @@ ## Quick Start - Unpack a firmware file and display it in the GUI !!! warning - OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! + OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. ```bash pip install ofrak From c39748e3ecfed4cb18750bc1fdafea3cbb99c132 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sun, 11 Feb 2024 21:14:05 -0500 Subject: [PATCH 09/25] Document the requirement of Python 3.8+ --- docs/environment-setup.md | 2 +- docs/getting-started.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/environment-setup.md b/docs/environment-setup.md index bb4a66aec..075949d7e 100644 --- a/docs/environment-setup.md +++ b/docs/environment-setup.md @@ -1,7 +1,7 @@ # Environment Setup & Installing OFRAK !!! warning - OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. + OFRAK is a Python library supporting Python3.8 and up. First and foremost, make sure your Python and pip installations are for Python3.8+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. There are three main ways one can set up an environment to use OFRAK: diff --git a/docs/getting-started.md b/docs/getting-started.md index 8716d56c2..0656d5449 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -3,7 +3,7 @@ ## Quick Start - Unpack a firmware file and display it in the GUI !!! warning - OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. + OFRAK is a Python library supporting Python3.8 and up. First and foremost, make sure your Python and pip installations are for Python3.8+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. ```bash pip install ofrak From c8d22c0d5f95cd07cf939c9d442cae05946808cc Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sun, 11 Feb 2024 21:15:13 -0500 Subject: [PATCH 10/25] Switch to angr 9.2.77 --- disassemblers/ofrak_angr/CHANGELOG.md | 2 +- disassemblers/ofrak_angr/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/disassemblers/ofrak_angr/CHANGELOG.md b/disassemblers/ofrak_angr/CHANGELOG.md index b36905321..d4ba9f23a 100644 --- a/disassemblers/ofrak_angr/CHANGELOG.md +++ b/disassemblers/ofrak_angr/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) ### Changed -- Update to latest angr==9.2.89, which also necessitates Python >= 3.8. +- Update to latest angr==9.2.77, which also necessitates Python >= 3.8. ### Fixed - Add `importlib-resources` dependency as workaround for z3-solver dependency issue. ([#401](https://github.com/redballoonsecurity/ofrak/pull/401)) diff --git a/disassemblers/ofrak_angr/requirements.txt b/disassemblers/ofrak_angr/requirements.txt index ca01977d1..c6a33a3dd 100644 --- a/disassemblers/ofrak_angr/requirements.txt +++ b/disassemblers/ofrak_angr/requirements.txt @@ -1,2 +1,2 @@ -angr==9.2.89 +angr==9.2.77 importlib-resources # A workaround for https://github.com/redballoonsecurity/ofrak/issues/398 From 8f59c226c7e1c9683a049a21e46376d47ff6b69d Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sun, 11 Feb 2024 22:24:13 -0500 Subject: [PATCH 11/25] `ofrak_core` also needs `pytest<8.0` --- ofrak_core/requirements-test.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ofrak_core/requirements-test.txt b/ofrak_core/requirements-test.txt index 5a863960d..977de5a37 100644 --- a/ofrak_core/requirements-test.txt +++ b/ofrak_core/requirements-test.txt @@ -1,5 +1,6 @@ autoflake==1.4 -pytest +# pytest-lazy-fixture does not work with pytest 8.0.0 - https://github.com/TvoroG/pytest-lazy-fixture/issues/65 +pytest<8.0 hypothesis~=6.39.3 hypothesis-trio trio-asyncio From 330e46ae98b50b83cdf8f30e8b6ca77205497fe0 Mon Sep 17 00:00:00 2001 From: Edward Larson Date: Fri, 16 Feb 2024 19:21:55 -0500 Subject: [PATCH 12/25] ignore DataWord in test due to angr bug --- .../ofrak_angr/ofrak_angr_test/test_unpackers.py | 8 ++++++++ .../pytest_ofrak/patterns/complex_block_unpacker.py | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py b/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py index 9b27d59ff..1997dca6d 100755 --- a/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py +++ b/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py @@ -1,6 +1,7 @@ from typing import Dict import pytest +from ofrak.core import DataWord from ofrak.core.basic_block import BasicBlock from pytest_ofrak.patterns.code_region_unpacker import ( @@ -62,6 +63,13 @@ async def expected_results(self, unpack_verify_test_case: ComplexBlockUnpackerTe pie_base_vaddr=0x400000, ) + elif unpack_verify_test_case.binary_md5_digest == "c79d1bea0398d7a9d0faa1ba68786f5e": + # Latest version of angr misses this DataWord now = the ref to it does not appear in the list of xrefs + unpack_verify_test_case.expected_results[0x8018] = [ + block for block in unpack_verify_test_case.expected_results[0x8018] + if not isinstance(block, DataWord) + ] + return unpack_verify_test_case.expected_results def _split_bb( diff --git a/ofrak_core/pytest_ofrak/patterns/complex_block_unpacker.py b/ofrak_core/pytest_ofrak/patterns/complex_block_unpacker.py index c632017d1..198e8391a 100644 --- a/ofrak_core/pytest_ofrak/patterns/complex_block_unpacker.py +++ b/ofrak_core/pytest_ofrak/patterns/complex_block_unpacker.py @@ -231,7 +231,7 @@ class ComplexBlockUnpackerTestCase(UnpackAndVerifyTestCase[int, List[Union[Basic is_exit_point=True, exit_vaddr=None, ), - DataWord(virtual_address=32816, size=4, format_string=" Date: Fri, 16 Feb 2024 19:31:17 -0500 Subject: [PATCH 13/25] add another now missing block --- .../ofrak_angr/ofrak_angr_test/test_unpackers.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py b/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py index 1997dca6d..656bcabfd 100755 --- a/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py +++ b/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py @@ -65,10 +65,17 @@ async def expected_results(self, unpack_verify_test_case: ComplexBlockUnpackerTe elif unpack_verify_test_case.binary_md5_digest == "c79d1bea0398d7a9d0faa1ba68786f5e": # Latest version of angr misses this DataWord now = the ref to it does not appear in the list of xrefs - unpack_verify_test_case.expected_results[0x8018] = [ - block for block in unpack_verify_test_case.expected_results[0x8018] - if not isinstance(block, DataWord) - ] + + missing_data_words = {0x8030, 0x8060} + + fixed_up_results = { + vaddr: [ + block for block in original_expected_blocks + if block.virtual_address not in missing_data_words + ] for vaddr, original_expected_blocks in unpack_verify_test_case.expected_results.items() + } + + return fixed_up_results return unpack_verify_test_case.expected_results From 300e29f8b50222c768adfc2a1a8ce9df7444e1d4 Mon Sep 17 00:00:00 2001 From: Edward Larson Date: Fri, 16 Feb 2024 19:32:18 -0500 Subject: [PATCH 14/25] black linting --- disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py b/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py index 656bcabfd..0a9905368 100755 --- a/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py +++ b/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py @@ -1,7 +1,6 @@ from typing import Dict import pytest -from ofrak.core import DataWord from ofrak.core.basic_block import BasicBlock from pytest_ofrak.patterns.code_region_unpacker import ( @@ -70,9 +69,11 @@ async def expected_results(self, unpack_verify_test_case: ComplexBlockUnpackerTe fixed_up_results = { vaddr: [ - block for block in original_expected_blocks + block + for block in original_expected_blocks if block.virtual_address not in missing_data_words - ] for vaddr, original_expected_blocks in unpack_verify_test_case.expected_results.items() + ] + for vaddr, original_expected_blocks in unpack_verify_test_case.expected_results.items() } return fixed_up_results From 2bd7547a470f52e2cd2a68633344107eedbdaeed Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sat, 17 Feb 2024 14:41:56 -0500 Subject: [PATCH 15/25] Attempt to fix a capstone error --- ofrak_core/pytest_ofrak/patterns/basic_block_unpacker.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ofrak_core/pytest_ofrak/patterns/basic_block_unpacker.py b/ofrak_core/pytest_ofrak/patterns/basic_block_unpacker.py index 5c0e0f012..e25858c39 100644 --- a/ofrak_core/pytest_ofrak/patterns/basic_block_unpacker.py +++ b/ofrak_core/pytest_ofrak/patterns/basic_block_unpacker.py @@ -756,6 +756,14 @@ class BasicBlockUnpackerTestCase( operands="", mode=InstructionSetMode.NONE, ), + Instruction( + virtual_address=0x4004E0, + size=2, + disassembly="repz ret ", + mnemonic="repz ret", + operands="", + mode=InstructionSetMode.NONE, + ), ), ], 0x4004F0: [ From 3801736c6058381448a1963988c048c095c85f27 Mon Sep 17 00:00:00 2001 From: rbs-alexr <122491504+rbs-alexr@users.noreply.github.com> Date: Tue, 13 Feb 2024 17:08:11 -0500 Subject: [PATCH 16/25] Dropping the .altinstr_replacement section from the toolchain (#414) * Dropping the .altinstr_replacement section from the toolchain * Updated CHANGELOG --- ofrak_patch_maker/CHANGELOG.md | 1 + ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py | 1 + 2 files changed, 2 insertions(+) diff --git a/ofrak_patch_maker/CHANGELOG.md b/ofrak_patch_maker/CHANGELOG.md index 17f7a66c7..f6cc95406 100644 --- a/ofrak_patch_maker/CHANGELOG.md +++ b/ofrak_patch_maker/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Fixed - X86_64 toolchain now installs on Docker image builds for AARCH64 hosts. ([#405](https://github.com/redballoonsecurity/ofrak/pull/405)) +- Toolchain now drops the .altinstrs_replacement as well as the .altinstructions section in our generated linker scripts ([#414](https://github.com/redballoonsecurity/ofrak/pull/414)) ## [4.0.2](https://github.com/redballoonsecurity/ofrak/compare/ofrak-patch-maker-v.4.0.1...ofrak-patch-maker-v.4.0.2) ### Fixed diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py index 61553bb32..9f437a482 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py @@ -95,6 +95,7 @@ def __init__( ".dynstr", ".eh_frame", ".altinstructions", + ".altinstr_replacement", ] self._assembler_target = self._get_assembler_target(processor) From a73db9614d7171b698bc9ab3109622f3d69c6103 Mon Sep 17 00:00:00 2001 From: Jacob Strieb <99368685+rbs-jacob@users.noreply.github.com> Date: Wed, 14 Feb 2024 15:38:41 -0500 Subject: [PATCH 17/25] Set the fallback font to monospace (#422) * Set the fallback font to monospace * Update CHANGELOG --- frontend/public/global.css | 3 +-- ofrak_core/CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/public/global.css b/frontend/public/global.css index 28171ad0e..b77e058d0 100644 --- a/frontend/public/global.css +++ b/frontend/public/global.css @@ -13,8 +13,7 @@ * { scrollbar-color: var(--main-fg-color) var(--main-bg-color); - font-family: var(--font), -apple-system, BlinkMacSystemFont, "Segoe UI", - Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif; + font-family: var(--font), monospace, monospace; font-variant-ligatures: none; } diff --git a/ofrak_core/CHANGELOG.md b/ofrak_core/CHANGELOG.md index 044532b50..fb12a9041 100644 --- a/ofrak_core/CHANGELOG.md +++ b/ofrak_core/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fix dragging and dropping in the GUI. ([#407](https://github.com/redballoonsecurity/ofrak/pull/407)) - Fix running scripts without a project selected, and without a config selected. ([#407](https://github.com/redballoonsecurity/ofrak/pull/407)) - Fix bug in OFRAK GUI server which causes an error when parsing a default config value of bytes. ([#409](https://github.com/redballoonsecurity/ofrak/pull/409)) +- Set default fallback font to system default monospace, instead of variable-width sans-serif. ([#422](https://github.com/redballoonsecurity/ofrak/pull/422)) ### Changed - Change `FreeSpaceModifier` & `PartialFreeSpaceModifier` behavior: an optional stub that isn't free space can be provided and fill-bytes for free space can be specified. ([#409](https://github.com/redballoonsecurity/ofrak/pull/409)) From 2a9a34401e7627ae2c5de9ff1609a68f9b9c074c Mon Sep 17 00:00:00 2001 From: Jacob Strieb <99368685+rbs-jacob@users.noreply.github.com> Date: Fri, 16 Feb 2024 14:52:17 -0500 Subject: [PATCH 18/25] Display strings with numbers primarily as strings (#423) * Display strings with numbers primarily as strings * Update CHANGELOG --- frontend/src/utils/StructuredList.svelte | 7 +++---- ofrak_core/CHANGELOG.md | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/utils/StructuredList.svelte b/frontend/src/utils/StructuredList.svelte index b36335dc7..45a9dcf02 100644 --- a/frontend/src/utils/StructuredList.svelte +++ b/frontend/src/utils/StructuredList.svelte @@ -9,10 +9,6 @@ } return Object.fromEntries(obj); - } else if (typeof object === "string") { - if (/^-?\d+$/.test(obj)) { - return parseInt(obj); - } } } @@ -37,6 +33,9 @@ {object} {:else if typeof object === "string" && !noQuotes} "{object}" + {#if /^-?\d+$/.test(object)} + (0x{parseInt(object).toString(16)}) + {/if} {:else if typeof object === "object" && Object.keys(object).length === 0} None {:else if typeof object === "object"} diff --git a/ofrak_core/CHANGELOG.md b/ofrak_core/CHANGELOG.md index fb12a9041..7ee5c6b09 100644 --- a/ofrak_core/CHANGELOG.md +++ b/ofrak_core/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fix running scripts without a project selected, and without a config selected. ([#407](https://github.com/redballoonsecurity/ofrak/pull/407)) - Fix bug in OFRAK GUI server which causes an error when parsing a default config value of bytes. ([#409](https://github.com/redballoonsecurity/ofrak/pull/409)) - Set default fallback font to system default monospace, instead of variable-width sans-serif. ([#422](https://github.com/redballoonsecurity/ofrak/pull/422)) +- View resource attribute string values containing only digits primarily as strings, alternatively as hex numbers. ([#423](https://github.com/redballoonsecurity/ofrak/pull/423)) ### Changed - Change `FreeSpaceModifier` & `PartialFreeSpaceModifier` behavior: an optional stub that isn't free space can be provided and fill-bytes for free space can be specified. ([#409](https://github.com/redballoonsecurity/ofrak/pull/409)) From 86a0445239ca400530cf5b1f27cc8da753bae741 Mon Sep 17 00:00:00 2001 From: Paul Noalhyt Date: Sat, 17 Feb 2024 02:16:13 +0100 Subject: [PATCH 19/25] Add typing support to ofrak_ghidra package (#421) * Add typing to ofrak_ghidra package * Add changelog --------- Co-authored-by: Paul Noalhyt --- disassemblers/ofrak_ghidra/CHANGELOG.md | 14 ++++++++++++++ disassemblers/ofrak_ghidra/ofrak_ghidra/py.typed | 0 disassemblers/ofrak_ghidra/setup.py | 1 + 3 files changed, 15 insertions(+) create mode 100644 disassemblers/ofrak_ghidra/CHANGELOG.md create mode 100644 disassemblers/ofrak_ghidra/ofrak_ghidra/py.typed diff --git a/disassemblers/ofrak_ghidra/CHANGELOG.md b/disassemblers/ofrak_ghidra/CHANGELOG.md new file mode 100644 index 000000000..f5bcdcb67 --- /dev/null +++ b/disassemblers/ofrak_ghidra/CHANGELOG.md @@ -0,0 +1,14 @@ +# Changelog +All notable changes to `ofrak-ghidra` will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) + +## 0.1.1 - 2024-02-15 +### Added +- Added typing support to the ofrak-ghidra package. This is helpful for users who use `mypy` and `ofrak_ghidra` in a project. + +## 0.1.0 - 2022-08-09 +### Added +Initial release. Hello world! diff --git a/disassemblers/ofrak_ghidra/ofrak_ghidra/py.typed b/disassemblers/ofrak_ghidra/ofrak_ghidra/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/disassemblers/ofrak_ghidra/setup.py b/disassemblers/ofrak_ghidra/setup.py index 0cd529cf3..a0f2eea93 100644 --- a/disassemblers/ofrak_ghidra/setup.py +++ b/disassemblers/ofrak_ghidra/setup.py @@ -37,6 +37,7 @@ def read_requirements(requirements_path): description="OFRAK Ghidra Components", url="", # TODO packages=setuptools.find_packages(), + package_data={"ofrak_ghidra": ["py.typed"]}, classifiers=[ "Programming Language :: Python :: 3", "Operating System :: OS Independent", From 9701ba7df33f20c135d75cc74de8ac812f5bdff7 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sat, 17 Feb 2024 15:10:32 -0500 Subject: [PATCH 20/25] Increase time limit on `test_comment_content` --- ofrak_core/test_ofrak/components/test_comments.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ofrak_core/test_ofrak/components/test_comments.py b/ofrak_core/test_ofrak/components/test_comments.py index 4e8fee044..085b11c1b 100644 --- a/ofrak_core/test_ofrak/components/test_comments.py +++ b/ofrak_core/test_ofrak/components/test_comments.py @@ -1,3 +1,4 @@ +from datetime import timedelta import pytest from hypothesis import given, HealthCheck, settings from hypothesis.strategies import text @@ -38,7 +39,10 @@ async def test_adding_comments(executable_resource: Resource): # We suppress the function_scoped_fixture health check because the executable_resource fixture # doesn't need to be reset between individual runs of hypothesis (since the comment overrides # the previous one every time). -@settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) +@settings( + suppress_health_check=[HealthCheck.function_scoped_fixture], + deadline=timedelta(seconds=5), +) @given(comment_str=text()) async def test_comment_content(executable_resource: Resource, comment_str: str): """Test comments with all kinds of string contents.""" From 1249ed80cba8cc00c3fa577e7a8c5f0934ccdd48 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sat, 17 Feb 2024 16:31:55 -0500 Subject: [PATCH 21/25] Fix a spurious "no current event loop" test error --- ofrak_core/test_ofrak/unit/test_ofrak_context.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ofrak_core/test_ofrak/unit/test_ofrak_context.py b/ofrak_core/test_ofrak/unit/test_ofrak_context.py index ed50f2bb7..93eb23ca9 100644 --- a/ofrak_core/test_ofrak/unit/test_ofrak_context.py +++ b/ofrak_core/test_ofrak/unit/test_ofrak_context.py @@ -64,6 +64,9 @@ async def run_component_with_installed_dependency(ofrak_context: OFRAKContext): def test_get_ofrak_context_over_time(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + # No active context before running OFRAK with pytest.raises(InvalidStateError): get_current_ofrak_context() From f019d486d33d4cd5edfe8e519588a29bdd2a317c Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Tue, 20 Feb 2024 19:15:39 -0500 Subject: [PATCH 22/25] Explain 3.7 vs 3.8 better in the docs --- docs/environment-setup.md | 2 +- docs/getting-started.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/environment-setup.md b/docs/environment-setup.md index 075949d7e..7e6e45396 100644 --- a/docs/environment-setup.md +++ b/docs/environment-setup.md @@ -1,7 +1,7 @@ # Environment Setup & Installing OFRAK !!! warning - OFRAK is a Python library supporting Python3.8 and up. First and foremost, make sure your Python and pip installations are for Python3.8+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. + OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with, and some components require Python >=3.8. There are three main ways one can set up an environment to use OFRAK: diff --git a/docs/getting-started.md b/docs/getting-started.md index 0656d5449..f31d8bacf 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -3,7 +3,7 @@ ## Quick Start - Unpack a firmware file and display it in the GUI !!! warning - OFRAK is a Python library supporting Python3.8 and up. First and foremost, make sure your Python and pip installations are for Python3.8+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. + OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with, and some components require Python >=3.8. ```bash pip install ofrak From 5237d9a95d81bc8b2f9511701764ccc5478c8bbe Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Tue, 20 Feb 2024 19:36:13 -0500 Subject: [PATCH 23/25] Cite specific versions of angr in comment --- disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py b/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py index 0a9905368..7e9279d31 100755 --- a/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py +++ b/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py @@ -63,7 +63,8 @@ async def expected_results(self, unpack_verify_test_case: ComplexBlockUnpackerTe ) elif unpack_verify_test_case.binary_md5_digest == "c79d1bea0398d7a9d0faa1ba68786f5e": - # Latest version of angr misses this DataWord now = the ref to it does not appear in the list of xrefs + # Unlike angr 9.2.6, angr 9.2.77 and 9.2.91 miss this DataWord now + # = the ref to it does not appear in the list of xrefs missing_data_words = {0x8030, 0x8060} From 7634cb2d64d7f9de09fcc9649657088fe9aff026 Mon Sep 17 00:00:00 2001 From: Wyatt <53830972+whyitfor@users.noreply.github.com> Date: Thu, 22 Feb 2024 14:28:03 -0500 Subject: [PATCH 24/25] Update docs/environment-setup.md --- docs/environment-setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/environment-setup.md b/docs/environment-setup.md index 7e6e45396..a38bd2bc7 100644 --- a/docs/environment-setup.md +++ b/docs/environment-setup.md @@ -1,7 +1,7 @@ # Environment Setup & Installing OFRAK !!! warning - OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with, and some components require Python >=3.8. + OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with, and some packages (for example, ofrak-angr) require Python >=3.8. There are three main ways one can set up an environment to use OFRAK: From 64518d511ffc041e0676c8a07bf38eef0664b8d4 Mon Sep 17 00:00:00 2001 From: Wyatt <53830972+whyitfor@users.noreply.github.com> Date: Thu, 22 Feb 2024 14:28:13 -0500 Subject: [PATCH 25/25] Update docs/getting-started.md --- docs/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index f31d8bacf..25d7ebec7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -3,7 +3,7 @@ ## Quick Start - Unpack a firmware file and display it in the GUI !!! warning - OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with, and some components require Python >=3.8. + OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with, and some packages (for example, ofrak-angr) require Python >=3.8. ```bash pip install ofrak