From ac02665ff730efe830f82dcfbaa50110ee5ad927 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Tue, 14 Jan 2025 14:44:32 -0500 Subject: [PATCH 01/11] Fix circular dependencies by making `ofrak_angr` and `ofrak_capstone` optional for `ofrak_core` tests Fixes #420 --- .github/workflows/test-all.yml | 44 +++---------------- ofrak_core/setup.py | 12 +---- .../components/test_patch_from_source.py | 4 +- .../components/test_symbolic_analysis.py | 2 +- 4 files changed, 10 insertions(+), 52 deletions(-) diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index cae4522d7..2c787cf22 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -50,8 +50,8 @@ jobs: cd frontend make check - ofrak-ghidra: - name: Test main OFRAK components + ofrak-dev: + name: Test all OFRAK components runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 @@ -62,11 +62,11 @@ jobs: python-version: '3.7' cache: pip cache-dependency-path: '**/setup.py' - - name: Build Ghidra image + - name: Build Dev image run: | python3 -m pip install PyYAML python3 build_image.py \ - --config ofrak-ghidra.yml \ + --config ofrak-dev.yml \ --base \ --finish \ --cache-from redballoonsecurity/ofrak/core-dev-base:latest @@ -77,7 +77,7 @@ jobs: --rm \ --entrypoint bash \ --volume "$(pwd)":/ofrak \ - redballoonsecurity/ofrak/ghidra:latest \ + redballoonsecurity/ofrak/dev:latest \ -c "cd /ofrak \ && mkdocs build --site-dir /tmp/docs" - name: Test components @@ -86,43 +86,11 @@ jobs: --interactive \ --rm \ --entrypoint bash \ - redballoonsecurity/ofrak/ghidra:latest \ + redballoonsecurity/ofrak/dev:latest \ -c "python -m ofrak_ghidra.server start \ && ofrak license --community --i-agree \ && make test" - ofrak-angr: - name: Test OFRAK angr and capstone components - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - with: - lfs: true - - uses: actions/setup-python@v4 - with: - python-version: '3.7' - cache: pip - cache-dependency-path: '**/setup.py' - - name: Build angr image - run: | - python3 -m pip install PyYAML - python3 build_image.py \ - --config ofrak-angr.yml \ - --base \ - --finish \ - --cache-from redballoonsecurity/ofrak/core-dev-base:latest - - name: Test components - run: | - docker run \ - --interactive \ - --rm \ - --entrypoint bash \ - --volume "$(pwd)":/ofrak \ - redballoonsecurity/ofrak/angr:latest \ - -c "ofrak license --community --i-agree \ - && make -C /ofrak_angr test \ - && make -C /ofrak_capstone test" - ofrak-tutorial: name: Test OFRAK examples and tutorial notebooks runs-on: ubuntu-22.04 diff --git a/ofrak_core/setup.py b/ofrak_core/setup.py index 067053a0b..2fa113a41 100644 --- a/ofrak_core/setup.py +++ b/ofrak_core/setup.py @@ -83,17 +83,7 @@ def read_requirements(requirements_path): + read_requirements("requirements.txt"), extras_require={ "docs": read_requirements("requirements-docs.txt"), - "test": [ - "importlib-resources", # Needed because of https://github.com/redballoonsecurity/ofrak/issues/398 - "ofrak_angr~=1.0", - # Running the tests requires this version of capstone. - # If building a Docker image, it should already be installed on the system. - # If you are running the tests in another environment, you will need to make sure - # that this version is installed from GitHub, since it depends on ofrak==3.3.0rc0 - # and neither of these are yet released on PyPI. - # "ofrak_capstone>=1.1.0rc0", - ] - + read_requirements("requirements-test.txt"), + "test": read_requirements("requirements-test.txt"), "non-pypi": read_requirements("requirements-non-pypi.txt"), }, author="Red Balloon Security", diff --git a/ofrak_core/test_ofrak/components/test_patch_from_source.py b/ofrak_core/test_ofrak/components/test_patch_from_source.py index 1577d7891..b3f0872ee 100644 --- a/ofrak_core/test_ofrak/components/test_patch_from_source.py +++ b/ofrak_core/test_ofrak/components/test_patch_from_source.py @@ -4,8 +4,8 @@ from ofrak_patch_maker.toolchain.llvm_12 import LLVM_12_0_1_Toolchain -import ofrak_angr -import ofrak_capstone +ofrak_angr = pytest.importorskip("ofrak_angr") +ofrak_capstone = pytest.importorskip("ofrak_capstone") from ofrak import OFRAKContext, Resource, ResourceAttributeValueFilter, ResourceFilter from ofrak.core import ( Allocatable, diff --git a/ofrak_core/test_ofrak/components/test_symbolic_analysis.py b/ofrak_core/test_ofrak/components/test_symbolic_analysis.py index 1d6dc7574..cc16108b3 100644 --- a/ofrak_core/test_ofrak/components/test_symbolic_analysis.py +++ b/ofrak_core/test_ofrak/components/test_symbolic_analysis.py @@ -3,7 +3,7 @@ import pytest -import ofrak_angr +ofrak_angr = pytest.importorskip("ofrak_angr") from ofrak import OFRAKContext, Resource, ResourceFilter from ofrak.core import ( ElfSymbolType, From a6cd200b100f38695e3166ebb316b45840327199 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Tue, 14 Jan 2025 15:02:24 -0500 Subject: [PATCH 02/11] Use ofrak-angr-ghidra.yml for all github CI tests --- .github/workflows/test-all.yml | 8 ++++---- ofrak-angr-ghidra.yml | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 ofrak-angr-ghidra.yml diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index 2c787cf22..eaf693546 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -50,7 +50,7 @@ jobs: cd frontend make check - ofrak-dev: + ofrak-angr-ghidra: name: Test all OFRAK components runs-on: ubuntu-22.04 steps: @@ -66,7 +66,7 @@ jobs: run: | python3 -m pip install PyYAML python3 build_image.py \ - --config ofrak-dev.yml \ + --config ofrak-angr-ghidra.yml \ --base \ --finish \ --cache-from redballoonsecurity/ofrak/core-dev-base:latest @@ -77,7 +77,7 @@ jobs: --rm \ --entrypoint bash \ --volume "$(pwd)":/ofrak \ - redballoonsecurity/ofrak/dev:latest \ + redballoonsecurity/ofrak/angr-ghidra:latest \ -c "cd /ofrak \ && mkdocs build --site-dir /tmp/docs" - name: Test components @@ -86,7 +86,7 @@ jobs: --interactive \ --rm \ --entrypoint bash \ - redballoonsecurity/ofrak/dev:latest \ + redballoonsecurity/ofrak/angr-ghidra:latest \ -c "python -m ofrak_ghidra.server start \ && ofrak license --community --i-agree \ && make test" diff --git a/ofrak-angr-ghidra.yml b/ofrak-angr-ghidra.yml new file mode 100644 index 000000000..bc158f89c --- /dev/null +++ b/ofrak-angr-ghidra.yml @@ -0,0 +1,18 @@ +registry: "redballoonsecurity/ofrak" +base_image_name: "angr-ghidra-base" +image_name: "angr-ghidra" +packages_paths: + [ + "ofrak_type", + "ofrak_io", + "ofrak_patch_maker", + "ofrak_core", + "disassemblers/ofrak_capstone", + "disassemblers/ofrak_angr", + "disassemblers/ofrak_ghidra", + "frontend", + ] +entrypoint: | + nginx \ + & python3 -m ofrak_ghidra.server start \ + & python3 -m ofrak gui -H 0.0.0.0 -p 8877 --backend ghidra From f07b2f7bb76c389d80abca6ef040bcf136c071c9 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Wed, 15 Jan 2025 11:36:45 -0500 Subject: [PATCH 03/11] Revert "Use ofrak-angr-ghidra.yml for all github CI tests" This reverts commit a6cd200b100f38695e3166ebb316b45840327199. --- .github/workflows/test-all.yml | 8 ++++---- ofrak-angr-ghidra.yml | 18 ------------------ 2 files changed, 4 insertions(+), 22 deletions(-) delete mode 100644 ofrak-angr-ghidra.yml diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index eaf693546..2c787cf22 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -50,7 +50,7 @@ jobs: cd frontend make check - ofrak-angr-ghidra: + ofrak-dev: name: Test all OFRAK components runs-on: ubuntu-22.04 steps: @@ -66,7 +66,7 @@ jobs: run: | python3 -m pip install PyYAML python3 build_image.py \ - --config ofrak-angr-ghidra.yml \ + --config ofrak-dev.yml \ --base \ --finish \ --cache-from redballoonsecurity/ofrak/core-dev-base:latest @@ -77,7 +77,7 @@ jobs: --rm \ --entrypoint bash \ --volume "$(pwd)":/ofrak \ - redballoonsecurity/ofrak/angr-ghidra:latest \ + redballoonsecurity/ofrak/dev:latest \ -c "cd /ofrak \ && mkdocs build --site-dir /tmp/docs" - name: Test components @@ -86,7 +86,7 @@ jobs: --interactive \ --rm \ --entrypoint bash \ - redballoonsecurity/ofrak/angr-ghidra:latest \ + redballoonsecurity/ofrak/dev:latest \ -c "python -m ofrak_ghidra.server start \ && ofrak license --community --i-agree \ && make test" diff --git a/ofrak-angr-ghidra.yml b/ofrak-angr-ghidra.yml deleted file mode 100644 index bc158f89c..000000000 --- a/ofrak-angr-ghidra.yml +++ /dev/null @@ -1,18 +0,0 @@ -registry: "redballoonsecurity/ofrak" -base_image_name: "angr-ghidra-base" -image_name: "angr-ghidra" -packages_paths: - [ - "ofrak_type", - "ofrak_io", - "ofrak_patch_maker", - "ofrak_core", - "disassemblers/ofrak_capstone", - "disassemblers/ofrak_angr", - "disassemblers/ofrak_ghidra", - "frontend", - ] -entrypoint: | - nginx \ - & python3 -m ofrak_ghidra.server start \ - & python3 -m ofrak gui -H 0.0.0.0 -p 8877 --backend ghidra From 9e042af11a5f8534a68ae2e460c5bfb6c6f420d8 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Wed, 15 Jan 2025 11:44:22 -0500 Subject: [PATCH 04/11] Add `ofrak_angr` to `ofrak_ghidra` per Wyatt's guidance --- ofrak-ghidra.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ofrak-ghidra.yml b/ofrak-ghidra.yml index 293ed2a8a..dd11c7caa 100644 --- a/ofrak-ghidra.yml +++ b/ofrak-ghidra.yml @@ -8,6 +8,7 @@ packages_paths: "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_capstone", + "disassemblers/ofrak_angr", "disassemblers/ofrak_ghidra", "frontend", ] From a45b3835606ced7fe94719f0099287dea5ccc5e3 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Wed, 15 Jan 2025 11:45:54 -0500 Subject: [PATCH 05/11] Revert the workflow file to what it was originally --- .github/workflows/test-all.yml | 44 +++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index 2c787cf22..cae4522d7 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -50,8 +50,8 @@ jobs: cd frontend make check - ofrak-dev: - name: Test all OFRAK components + ofrak-ghidra: + name: Test main OFRAK components runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 @@ -62,11 +62,11 @@ jobs: python-version: '3.7' cache: pip cache-dependency-path: '**/setup.py' - - name: Build Dev image + - name: Build Ghidra image run: | python3 -m pip install PyYAML python3 build_image.py \ - --config ofrak-dev.yml \ + --config ofrak-ghidra.yml \ --base \ --finish \ --cache-from redballoonsecurity/ofrak/core-dev-base:latest @@ -77,7 +77,7 @@ jobs: --rm \ --entrypoint bash \ --volume "$(pwd)":/ofrak \ - redballoonsecurity/ofrak/dev:latest \ + redballoonsecurity/ofrak/ghidra:latest \ -c "cd /ofrak \ && mkdocs build --site-dir /tmp/docs" - name: Test components @@ -86,11 +86,43 @@ jobs: --interactive \ --rm \ --entrypoint bash \ - redballoonsecurity/ofrak/dev:latest \ + redballoonsecurity/ofrak/ghidra:latest \ -c "python -m ofrak_ghidra.server start \ && ofrak license --community --i-agree \ && make test" + ofrak-angr: + name: Test OFRAK angr and capstone components + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + with: + lfs: true + - uses: actions/setup-python@v4 + with: + python-version: '3.7' + cache: pip + cache-dependency-path: '**/setup.py' + - name: Build angr image + run: | + python3 -m pip install PyYAML + python3 build_image.py \ + --config ofrak-angr.yml \ + --base \ + --finish \ + --cache-from redballoonsecurity/ofrak/core-dev-base:latest + - name: Test components + run: | + docker run \ + --interactive \ + --rm \ + --entrypoint bash \ + --volume "$(pwd)":/ofrak \ + redballoonsecurity/ofrak/angr:latest \ + -c "ofrak license --community --i-agree \ + && make -C /ofrak_angr test \ + && make -C /ofrak_capstone test" + ofrak-tutorial: name: Test OFRAK examples and tutorial notebooks runs-on: ubuntu-22.04 From 0f6e8c149c87194e91e6bb8956601dac15c55cc7 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Wed, 15 Jan 2025 12:02:36 -0500 Subject: [PATCH 06/11] Do not insist on 100% functional coverage, unless ofrak_angr is there --- ofrak_core/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_core/Makefile b/ofrak_core/Makefile index 847b3cdd3..491e01059 100644 --- a/ofrak_core/Makefile +++ b/ofrak_core/Makefile @@ -18,7 +18,7 @@ test: inspect $(PYTHON) -m pytest -n auto test_ofrak --cov=ofrak --cov-report=term-missing (sleep 2; echo 1; sleep 2; echo i agree) \ | python3 -m coverage run --append --source ofrak -m ofrak license --force - fun-coverage --cov-fail-under=100 + if python3.9 -m pip show -qqq ofrak_angr; then fun-coverage --cov-fail-under=100; fi ofrak/gui/public: if [ -d /ofrak_gui ] ; then \ From 8474d00a4829c5f5679742f83518b0cf4ecb75a7 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Wed, 15 Jan 2025 12:28:20 -0500 Subject: [PATCH 07/11] Typo fix --- ofrak_core/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_core/Makefile b/ofrak_core/Makefile index 491e01059..39e56618c 100644 --- a/ofrak_core/Makefile +++ b/ofrak_core/Makefile @@ -18,7 +18,7 @@ test: inspect $(PYTHON) -m pytest -n auto test_ofrak --cov=ofrak --cov-report=term-missing (sleep 2; echo 1; sleep 2; echo i agree) \ | python3 -m coverage run --append --source ofrak -m ofrak license --force - if python3.9 -m pip show -qqq ofrak_angr; then fun-coverage --cov-fail-under=100; fi + if python3 -m pip show -qqq ofrak_angr; then fun-coverage --cov-fail-under=100; fi ofrak/gui/public: if [ -d /ofrak_gui ] ; then \ From 283ef6c72adca61ea278e715d8011f9a7161c25c Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Wed, 15 Jan 2025 12:32:08 -0500 Subject: [PATCH 08/11] Should be using the PYTHON variable --- ofrak_core/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_core/Makefile b/ofrak_core/Makefile index 39e56618c..06e04efb7 100644 --- a/ofrak_core/Makefile +++ b/ofrak_core/Makefile @@ -18,7 +18,7 @@ test: inspect $(PYTHON) -m pytest -n auto test_ofrak --cov=ofrak --cov-report=term-missing (sleep 2; echo 1; sleep 2; echo i agree) \ | python3 -m coverage run --append --source ofrak -m ofrak license --force - if python3 -m pip show -qqq ofrak_angr; then fun-coverage --cov-fail-under=100; fi + if $(PYTHON) -m pip show -qqq ofrak_angr; then fun-coverage --cov-fail-under=100; fi ofrak/gui/public: if [ -d /ofrak_gui ] ; then \ From 76ffeb9a00d17695a6b69207b3413dd76d64763c Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Wed, 15 Jan 2025 14:37:13 -0500 Subject: [PATCH 09/11] Put the commented requirements in, per Wyatt --- ofrak_core/setup.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ofrak_core/setup.py b/ofrak_core/setup.py index 2fa113a41..8e4398774 100644 --- a/ofrak_core/setup.py +++ b/ofrak_core/setup.py @@ -83,7 +83,18 @@ def read_requirements(requirements_path): + read_requirements("requirements.txt"), extras_require={ "docs": read_requirements("requirements-docs.txt"), - "test": read_requirements("requirements-test.txt"), + "test": [ + # Running the full set of tests requires these version of angr and capstone. + # If building a Docker image, it should already be installed on the system. + # If you are running the tests in another environment, you will need to make sure + # that this version is installed from GitHub, since it depends on ofrak==3.3.0rc0 + # and neither of these are yet released on PyPI. + # If these are not installed, some of the tests will be skipped + # "ofrak_capstone>=1.1.0rc0", + # "ofrak_angr~=1.1.0rc0", + # "importlib-resources", # Needed because of https://github.com/redballoonsecurity/ofrak/issues/398 + ] + + read_requirements("requirements-test.txt"), "non-pypi": read_requirements("requirements-non-pypi.txt"), }, author="Red Balloon Security", From 01330a662a4f5e9cd677e9db4187ee45007f1222 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Fri, 17 Jan 2025 09:48:57 -0500 Subject: [PATCH 10/11] Always enforce coverage Co-authored-by: Wyatt <53830972+whyitfor@users.noreply.github.com> --- ofrak_core/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_core/Makefile b/ofrak_core/Makefile index 06e04efb7..847b3cdd3 100644 --- a/ofrak_core/Makefile +++ b/ofrak_core/Makefile @@ -18,7 +18,7 @@ test: inspect $(PYTHON) -m pytest -n auto test_ofrak --cov=ofrak --cov-report=term-missing (sleep 2; echo 1; sleep 2; echo i agree) \ | python3 -m coverage run --append --source ofrak -m ofrak license --force - if $(PYTHON) -m pip show -qqq ofrak_angr; then fun-coverage --cov-fail-under=100; fi + fun-coverage --cov-fail-under=100 ofrak/gui/public: if [ -d /ofrak_gui ] ; then \ From 7cc46a2d40c23e3071760050423bb2e1943b3ee0 Mon Sep 17 00:00:00 2001 From: Wyatt Date: Fri, 17 Jan 2025 16:33:11 -0500 Subject: [PATCH 11/11] Add changelog entry, ofrak==3.3.0rc1 --- ofrak_core/CHANGELOG.md | 3 ++- ofrak_core/setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ofrak_core/CHANGELOG.md b/ofrak_core/CHANGELOG.md index 99b93508c..a1083416b 100644 --- a/ofrak_core/CHANGELOG.md +++ b/ofrak_core/CHANGELOG.md @@ -3,7 +3,7 @@ All notable changes to `ofrak` will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased: 3.3.0rc0](https://github.com/redballoonsecurity/ofrak/tree/master) +## [Unreleased: 3.3.0rc1](https://github.com/redballoonsecurity/ofrak/tree/master) ### Added - Add license check command to prompt users about community or pro licenses. ([#478](https://github.com/redballoonsecurity/ofrak/pull/478)) - Support `application/vnd.android.package-archive` mime type for APKs, which is returned by newer versions of libmagic ([#470](https://github.com/redballoonsecurity/ofrak/pull/470)) @@ -43,6 +43,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fix unintentional ignoring of cpio errors introduced in [#486](https://github.com/redballoonsecurity/ofrak/pull/486) ([#555](https://github.com/redballoonsecurity/ofrak/pull/555])) - `Data` resource attribute always corresponds to value of `Resource.get_data_range_within_root` ([#559](https://github.com/redballoonsecurity/ofrak/pull/559)) - Fixed endianness issue in DTB raw byte identifier ([#492](https://github.com/redballoonsecurity/ofrak/pull/492)) +- Make `ofrak[test]` dependencies on `ofrak-capstone` and `ofrak-angr` optional to avoid circular-dependency issues ([#571](https://github.com/redballoonsecurity/ofrak/pull/571)) ### Changed - By default, the ofrak log is now `ofrak-YYYYMMDDhhmmss.log` rather than just `ofrak.log` and the name can be specified on the command line ([#480](https://github.com/redballoonsecurity/ofrak/pull/480)) diff --git a/ofrak_core/setup.py b/ofrak_core/setup.py index 8e4398774..9d15489e8 100644 --- a/ofrak_core/setup.py +++ b/ofrak_core/setup.py @@ -69,7 +69,7 @@ def read_requirements(requirements_path): setuptools.setup( name="ofrak", - version="3.3.0rc0", + version="3.3.0rc1", description="A binary analysis and modification platform", packages=setuptools.find_packages(exclude=["test_ofrak", "test_ofrak.*"]), package_data={