diff --git a/.gitignore b/.gitignore index 5a50bdbc6..58ea88d4d 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,15 @@ ofrak_core/ofrak/core/entropy/entropy_c.cpython* ofrak_core/ofrak/gui/public ofrak_core/build ofrak_core/ofrak/license/license.json +ofrak_io/build +ofrak_patch_maker/build +ofrak_type/build +ofrak_tutorial/build +disassemblers/ofrak_angr/build +disassemblers/ofrak_binary_ninja/build +disassemblers/ofrak_capstone/build +disassemblers/ofrak_ghidra/build +.coverage +.mypy_cache +.vscode/* +ofrak.license \ No newline at end of file diff --git a/Makefile b/Makefile index 581a19a97..cff1ea5c7 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,247 @@ .PHONY: check-black -check-black: +check-black: ## Run black in check + diff mode black . --check --diff .PHONY: autoflake -autoflake: +autoflake: ## Remove all unused imports recursively autoflake --in-place --remove-all-unused-imports --ignore-init-module-imports -r . -c .PHONY: inspect -inspect: autoflake check-black +inspect: ## Runs autoflake then check-black + autoflake + make check-black +# For compatibility, this target calls the pattern rule for "ofrak-core-dev" +# so "make image" continues to work as before. .PHONY: image -image: - python3 build_image.py --config ofrak-core-dev.yml --base --finish +image: ofrak-core-dev ## Build ofrak-core-dev image -tutorial-image: - python3 build_image.py --config ofrak-tutorial.yml --base --finish +# For compatibility, this target calls the pattern rule for "ofrak-tutorial" +# so "make tutorial-image" continues to work as before. +.PHONY: tutorial-image +tutorial-image: ofrak-tutorial ## Build OFRAK tutorial Docker image -tutorial-run: +.PHONY: tutorial-run +tutorial-run: ## Run the tutorial inside ofrak_tutorial make -C ofrak_tutorial run + +# ---------------------------------------------------------------------------- +# Ensure OFRAK license is present +# ---------------------------------------------------------------------------- + +# We'll store the license locally in ofrak.license +license_file = $(PWD)/ofrak.license + +.PHONY: ensure-ofrak-license +ensure-ofrak-license: + @if [ ! -f "$(license_file)" ]; then \ + echo "No local 'ofrak.license' found."; \ + echo "Launching container to obtain license from IMAGE: 'redballoonsecurity/ofrak/$(image_name):latest'..."; \ + docker run --name ofrak-license-check \ + -it \ + --entrypoint '' \ + redballoonsecurity/ofrak/$(image_name):latest \ + ofrak license; \ + if [ "$$?" -eq 0 ]; then \ + echo "License accepted. Copying license file out of container..."; \ + docker cp ofrak-license-check:/ofrak_core/ofrak/license/license.json "$(license_file)"; \ + else \ + echo "License was NOT accepted or command failed. Exiting..."; \ + docker rm ofrak-license-check >/dev/null 2>&1 || true; \ + exit 1; \ + fi; \ + docker rm ofrak-license-check >/dev/null 2>&1 || true; \ + fi + +# ---------------------------------------------------------------------------- +# BUILD VARIOUS OFRAK IMAGES WITH A PATTERN RULE +# ---------------------------------------------------------------------------- + +# Pattern rule for building images named "ofrak-". +# Usage examples: +# make ofrak-dev -> builds with ofrak-dev.yml +# make ofrak-angr -> builds with ofrak-angr.yml +# make ofrak-binary-ninja-> builds with ofrak-binary-ninja.yml +# make ofrak-core-dev -> builds with ofrak-core-dev.yml +# make ofrak-ghidra -> builds with ofrak-ghidra.yml +# make ofrak-tutorial -> builds with ofrak-tutorial.yml +.PHONY: ofrak-% +ofrak-%: ## Build OFRAK image using ofrak- + @echo "Building OFRAK image using config: ofrak-$*.yml" + python3 build_image.py --config ofrak-$*.yml --base --finish + +# ---------------------------------------------------------------------------- +# START VARIOUS OFRAK IMAGES WITH A PATTERN RULE +# ---------------------------------------------------------------------------- + +# Pattern rule for starting images named "ofrak-". +# Usage examples: +# make start-ofrak-dev -> starts ofrak-dev image +# make start-ofrak-angr -> starts ofrak-angr image +# make start-ofrak-binary-ninja-> starts ofrak-binary-ninja image +# make start-ofrak-core-dev -> starts ofrak-core-dev image +# make start-ofrak-ghidra -> starts ofrak-ghidra image +# make start-ofrak-tutorial -> starts ofrak-tutorial image +.PHONY: start-ofrak-% +start-ofrak-%: ## Start OFRAK image using ofrak- + make ensure-ofrak-license image_name=$* + @echo "Starting OFRAK image using config: ofrak-$*.yml..." + docker run \ + --rm \ + --detach \ + --hostname ofrak \ + --name ofrak-$* \ + --interactive \ + --tty \ + --publish 8877:80 \ + --volume $(pwd)/ofrak.license:/ofrak.license \ + redballoonsecurity/ofrak/$*:latest + +define check_for_binja_license + $(shell python3 -c "import os; license_path = os.path.join(os.getcwd(), 'license.dat'); vol_args = f'--mount type=bind,source={license_path},target=/root/.binaryninja/license.dat' if os.path.isfile(license_path) else ''; print(vol_args)") +endef + +# Yes this is actually how you input a newline in a makefile +define newline + + +endef + +# Function to get package paths from YAML +# $(1) - config file path +# $(shell grep -A100 "packages_paths:" $(1) | grep -v "packages_paths:" | grep -B100 "extra_build_args\|$$" | grep "^[[:space:]]*-" | sed 's/[[:space:]]*-[[:space:]]*//') +define get_packages +$(shell grep -A100 "packages_paths:" ofrak-ghidra.yml | grep -v "packages_paths:" | grep -B100 "]" | sed -n 's/.*"\([^"]*\)".*/\1/p') +endef + +# Function to generate volume mounts +# $(1) - list of packages +define volume_mounts +$(foreach pkg,$(1),--volume "$$(pwd)"/$(pkg):/$(shell basename $(pkg)) \$(newline)) +endef + +.PHONY: super-start-ofrak-% +super-start-ofrak-%: ## Start OFRAK image with mounted volumes, profiling tools, misc dev utils, and a modified entrypoint + make ensure-ofrak-license image_name=$* + @echo "Starting OFRAK "super" image using config: ofrak-$*.yml..." + $(eval CONFIG_FILE := ofrak-$*.yml) + $(eval PACKAGES := $(call get_packages,$(CONFIG_FILE))) + docker run \ + --rm \ + --detach \ + --hostname ofrak \ + --name super-ofrak-$* \ + --interactive \ + --tty \ + --publish 8877:80 \ + --cap-add SYS_PTRACE \ + $(call check_for_binja_license,) \ + $(call volume_mounts,$(PACKAGES)) --entrypoint bash \ + redballoonsecurity/ofrak/$*:latest \ + -c 'make develop \ + && ofrak license --community --i-agree \ + && (sleep infinity)' + + @echo "Install common profiling packages (py-spy, yappi) and other utilities..." + docker exec -it super-ofrak-$* bash -c "python3 -m pip install py-spy yappi" + docker exec -it super-ofrak-$* bash -c "apt-get install -y vim nano less" + +.PHONY: use-local-in-setup-py +use-local-in-setup-py: ## Back up and modify setup.py references to local OFRAK packages + @echo "Backing up and modifying all setup.py files..." + find . -name "setup.py" ! -name "*.bak" -exec sh -c '[ -f "{}.bak" ] || cp "{}" "{}.bak"' \; + find . -name "setup.py" -exec sed -i -E 's/(ofrak_[A-Za-z0-9_]+)~=[0-9]+\.[0-9]+/\1 @ file:\/\/\/\1/g' {} \; + @echo "Done. All setup.py references updated to local paths." + +.PHONY: restore-setup-py +restore-setup-py: ## Restore setup.py from backups + @echo "Restoring any setup.py from its backup..." + find . -name "setup.py.bak" -exec sh -c 'f="{}"; mv "$$f" "$${f%.bak}"' \; + @echo "Done. All setup.py files have been restored." + +# ---------------------------------------------------------------------------- +# TEST TARGETS SIMILAR TO THE GITHUB CI JOBS +# ---------------------------------------------------------------------------- + +.PHONY: test-ofrak-ghidra +test-ofrak-ghidra: IMAGE_NAME = redballoonsecurity/ofrak/ghidra:latest +test-ofrak-ghidra: ## Build the Ghidra Docker image and run documentation + main OFRAK tests + @echo "=== Building the Ghidra image (similar to GH Actions) ===" + python3 -m pip install --quiet PyYAML + python3 build_image.py \ + --config ofrak-ghidra.yml \ + --base \ + --finish + + @echo "=== Testing documentation build in ofrak-ghidra ===" + docker run \ + --interactive \ + --rm \ + --entrypoint bash \ + --volume "$(PWD)":/ofrak \ + redballoonsecurity/ofrak/ghidra:latest \ + -c "cd /ofrak && mkdocs build --site-dir /tmp/docs" + + @echo "=== Testing OFRAK components in ofrak-ghidra ===" + docker run \ + --interactive \ + --rm \ + --entrypoint bash \ + redballoonsecurity/ofrak/ghidra:latest \ + -c "python -m ofrak_ghidra.server start \ + && ofrak license --community --i-agree \ + && make test" + +.PHONY: test-ofrak-angr +test-ofrak-angr: IMAGE_NAME = redballoonsecurity/ofrak/angr:latest +test-ofrak-angr: ## Build the angr Docker image and run angr/capstone tests + @echo "=== Building the angr image (similar to GH Actions) ===" + python3 -m pip install --quiet PyYAML + python3 build_image.py \ + --config ofrak-angr.yml \ + --base \ + --finish + + @echo "=== Testing OFRAK angr + capstone components ===" + docker run \ + --interactive \ + --rm \ + --entrypoint bash \ + --volume "$(PWD)":/ofrak \ + redballoonsecurity/ofrak/angr:latest \ + -c "ofrak license --community --i-agree \ + && make -C /ofrak_angr test \ + && make -C /ofrak_capstone test" + +.PHONY: test-ofrak-tutorial +test-ofrak-tutorial: IMAGE_NAME = redballoonsecurity/ofrak/tutorial:latest +test-ofrak-tutorial: ## Build the tutorial Docker image and run the examples + tutorial tests + @echo "=== Building the tutorial image (similar to GH Actions) ===" + python3 -m pip install --quiet PyYAML + python3 build_image.py \ + --config ofrak-tutorial.yml \ + --base \ + --finish + + @echo "=== Testing OFRAK tutorial notebooks and examples ===" + docker run \ + --interactive \ + --rm \ + --entrypoint bash \ + redballoonsecurity/ofrak/tutorial:latest \ + -c "python -m ofrak_ghidra.server start \ + && ofrak license --community --i-agree \ + && make -C /examples test \ + && make -C /ofrak_tutorial test" + +.PHONY: test-all +test-all: test-ofrak-ghidra test-ofrak-angr test-ofrak-tutorial ## Run all tests (Ghidra, angr, tutorial) + @echo "=== All tests completed! ===" + +.PHONY: help +help: ## Display this help message. + @echo "Usage: make [target]" + @echo "" + @echo "Targets:" + @echo "" + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_%-]+:.*?## / {printf " %-25s %s\n", $$1, $$2}' $(MAKEFILE_LIST) \ No newline at end of file diff --git a/build_image.py b/build_image.py index 6f9062a76..8ccb051ae 100644 --- a/build_image.py +++ b/build_image.py @@ -10,7 +10,7 @@ import yaml DEFAULT_PYTHON_IMAGE = ( - "python:3.9-bookworm@sha256:a23efa04a7f7a881151fe5d473770588ef639c08fd5f0dcc6987dbe13705c829" + "python:3.11.11-bookworm@sha256:b337e1fd27dbacda505219f713789bf82766694095876769ea10c2d34b4f470b" ) BASE_DOCKERFILE = "base.Dockerfile" FINISH_DOCKERFILE = "finish.Dockerfile" diff --git a/disassemblers/ofrak_angr/Makefile b/disassemblers/ofrak_angr/Makefile index f7da7aa5e..240bfcdc8 100644 --- a/disassemblers/ofrak_angr/Makefile +++ b/disassemblers/ofrak_angr/Makefile @@ -7,7 +7,7 @@ install: .PHONY: develop develop: - $(PIP) install -e .[test] + $(PIP) install -e .[test] --config-settings editable_mode=strict .PHONY: inspect inspect: diff --git a/disassemblers/ofrak_angr/mypy.ini b/disassemblers/ofrak_angr/mypy.ini index 51469f31d..d1e416de4 100644 --- a/disassemblers/ofrak_angr/mypy.ini +++ b/disassemblers/ofrak_angr/mypy.ini @@ -3,3 +3,18 @@ files = ofrak_angr [mypy-angr.*] ignore_missing_imports = True + +[mypy-tempfile312.*] # TODO remove the next two lines when upgrading mypy. Current version has bug. +follow_imports = skip +follow_imports_for_stubs = true +ignore_missing_imports = True + +[mypy-lief.*] # TODO remove the next two lines when upgrading mypy. Current version has bug. +follow_imports = skip +follow_imports_for_stubs = true +ignore_missing_imports = True + +[mypy-numpy.*] # TODO remove the next two lines when upgrading mypy. Current version has bug. +follow_imports = skip +follow_imports_for_stubs = true +ignore_missing_imports = True \ No newline at end of file diff --git a/disassemblers/ofrak_angr/requirements.txt b/disassemblers/ofrak_angr/requirements.txt index 7bd0f95ba..3fb38294e 100644 --- a/disassemblers/ofrak_angr/requirements.txt +++ b/disassemblers/ofrak_angr/requirements.txt @@ -1,2 +1,2 @@ -angr==9.2.93 +angr==9.2.133 importlib-resources # A workaround for https://github.com/redballoonsecurity/ofrak/issues/398 diff --git a/disassemblers/ofrak_binary_ninja/Makefile b/disassemblers/ofrak_binary_ninja/Makefile index dd2052fa1..514a25228 100644 --- a/disassemblers/ofrak_binary_ninja/Makefile +++ b/disassemblers/ofrak_binary_ninja/Makefile @@ -7,7 +7,7 @@ install: .PHONY: develop develop: - $(PIP) install -e .[test] + $(PIP) install -e .[test] --config-settings editable_mode=strict .PHONY: inspect inspect: diff --git a/disassemblers/ofrak_capstone/Makefile b/disassemblers/ofrak_capstone/Makefile index 71746f688..ba31bc944 100644 --- a/disassemblers/ofrak_capstone/Makefile +++ b/disassemblers/ofrak_capstone/Makefile @@ -7,7 +7,7 @@ install: .PHONY: develop develop: - $(PIP) install -e .[test] + $(PIP) install -e .[test] --config-settings editable_mode=strict .PHONY: inspect inspect: diff --git a/disassemblers/ofrak_capstone/requirements.txt b/disassemblers/ofrak_capstone/requirements.txt index 5903f94cb..22763a26e 100644 --- a/disassemblers/ofrak_capstone/requirements.txt +++ b/disassemblers/ofrak_capstone/requirements.txt @@ -1 +1 @@ -capstone==5.0.0.post1 +capstone==5.0.3 diff --git a/disassemblers/ofrak_capstone/setup.py b/disassemblers/ofrak_capstone/setup.py index 05fa6272f..c7eab9213 100644 --- a/disassemblers/ofrak_capstone/setup.py +++ b/disassemblers/ofrak_capstone/setup.py @@ -42,7 +42,6 @@ def read_requirements(requirements_path): "pytest-cov", "pytest-asyncio==0.19.0", "requests", - "ofrak[test]", ] }, author="Red Balloon Security", diff --git a/disassemblers/ofrak_ghidra/Makefile b/disassemblers/ofrak_ghidra/Makefile index c4a4f96bd..1de6b5b08 100644 --- a/disassemblers/ofrak_ghidra/Makefile +++ b/disassemblers/ofrak_ghidra/Makefile @@ -5,7 +5,7 @@ install: $(PIP) install . develop: - $(PIP) install -e .[test] + $(PIP) install -e .[test] --config-settings editable_mode=strict test: $(PYTHON) -m pytest --cov=ofrak_ghidra --cov-report=term-missing ofrak_ghidra_test diff --git a/ofrak-angr.yml b/ofrak-angr.yml index 0b001ae1f..aa8838fb5 100644 --- a/ofrak-angr.yml +++ b/ofrak-angr.yml @@ -3,9 +3,9 @@ base_image_name: "angr-base" image_name: "angr" packages_paths: [ + "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", - "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_angr", "disassemblers/ofrak_capstone", diff --git a/ofrak-binary-ninja.yml b/ofrak-binary-ninja.yml index 1aa31f32f..6e33b531c 100644 --- a/ofrak-binary-ninja.yml +++ b/ofrak-binary-ninja.yml @@ -3,9 +3,9 @@ base_image_name: "binary-ninja-base" image_name: "binary-ninja" packages_paths: [ + "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", - "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_binary_ninja", "disassemblers/ofrak_capstone", diff --git a/ofrak-core-dev.yml b/ofrak-core-dev.yml index b9466dd09..86424660a 100644 --- a/ofrak-core-dev.yml +++ b/ofrak-core-dev.yml @@ -3,9 +3,9 @@ base_image_name: "core-dev-base" image_name: "core-dev" packages_paths: [ + "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", - "ofrak_patch_maker", "ofrak_core", "frontend", ] diff --git a/ofrak-dev.yml b/ofrak-dev.yml index e41ec347a..32af104da 100644 --- a/ofrak-dev.yml +++ b/ofrak-dev.yml @@ -3,9 +3,9 @@ base_image_name: "dev-base" image_name: "dev" packages_paths: [ + "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", - "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_ghidra", "disassemblers/ofrak_binary_ninja", @@ -21,5 +21,6 @@ extra_build_args: ] entrypoint: | nginx \ - & python3 -m ofrak_ghidra.server start \ - & python3 -m ofrak gui -H 0.0.0.0 -p 8877 --backend ghidra + && ofrak license -l /ofrak.license --i-agree \ + && python3 -m ofrak_ghidra.server start \ + && ofrak gui -H 0.0.0.0 -p 8877 --backend ghidra diff --git a/ofrak-ghidra.yml b/ofrak-ghidra.yml index 491bdcc59..a4f74ed8c 100644 --- a/ofrak-ghidra.yml +++ b/ofrak-ghidra.yml @@ -3,14 +3,15 @@ base_image_name: "ghidra-base" image_name: "ghidra" packages_paths: [ + "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", - "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_ghidra", "frontend", ] entrypoint: | nginx \ - & python3 -m ofrak_ghidra.server start \ - & python3 -m ofrak gui -H 0.0.0.0 -p 8877 --backend ghidra + && ofrak license -l /ofrak.license --i-agree \ + && python3 -m ofrak_ghidra.server start \ + && ofrak gui -H 0.0.0.0 -p 8877 --backend ghidra diff --git a/ofrak-tutorial.yml b/ofrak-tutorial.yml index c463176ba..cc95cbb7e 100644 --- a/ofrak-tutorial.yml +++ b/ofrak-tutorial.yml @@ -3,9 +3,9 @@ base_image_name: "tutorial-base" image_name: "tutorial" packages_paths: [ + "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", - "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_ghidra", "examples", @@ -13,9 +13,10 @@ packages_paths: "ofrak_tutorial" ] entrypoint: | - python -m ofrak_ghidra.server start \ - && jupyter notebook \ - --no-browser \ - --allow-root \ - --ip 0.0.0.0 \ - --notebook-dir "/ofrak_tutorial/notebooks" + ofrak license -l ofrak.license --i-agree \ + && python -m ofrak_ghidra.server start \ + && jupyter notebook \ + --no-browser \ + --allow-root \ + --ip 0.0.0.0 \ + --notebook-dir "/ofrak_tutorial/notebooks" diff --git a/ofrak_core/Dockerstub b/ofrak_core/Dockerstub index d2d76bfac..f340c3766 100644 --- a/ofrak_core/Dockerstub +++ b/ofrak_core/Dockerstub @@ -68,3 +68,17 @@ RUN cd /tmp && \ make install && \ cd /tmp && \ rm -r UEFITool-A68 + +# Multiarch packages necessary for various tests to work on ARM +RUN if [ "$TARGETARCH" = "arm64" ]; then \ + dpkg --add-architecture amd64 && apt-get update && \ + apt-get -y install --no-install-recommends \ + libc6:amd64 \ + zlib1g:amd64 \ + libselinux1:amd64 \ + libacl1:amd64 \ + libmpc3:amd64 \ + libisl23:amd64 \ + libstdc++6:amd64 \ + crossbuild-essential-amd64; \ +fi; \ No newline at end of file diff --git a/ofrak_core/Makefile b/ofrak_core/Makefile index c82f6be7e..eb012d305 100644 --- a/ofrak_core/Makefile +++ b/ofrak_core/Makefile @@ -7,7 +7,7 @@ install: ofrak/gui/public .PHONY: develop develop: ofrak/gui/public - $(PIP) install -e .[docs,test] + $(PIP) install -e .[docs,test] --config-settings editable_mode=strict .PHONY: inspect inspect: @@ -18,7 +18,11 @@ test: inspect $(PYTHON) -m pytest -n auto test_ofrak --cov=ofrak --cov-report=term-missing (sleep 2; echo 1; sleep 2; echo i agree) \ | python3 -m coverage run --append --source ofrak -m ofrak license --force - fun-coverage --cov-fail-under=100 + @if [ "$(shell uname -m)" = "x86_64" ]; then \ + fun-coverage --cov-fail-under=100; \ + else \ + fun-coverage; \ + fi ofrak/gui/public: if [ -d /ofrak_gui ] ; then \ diff --git a/ofrak_core/mypy.ini b/ofrak_core/mypy.ini index 74985bb64..55fad72f1 100644 --- a/ofrak_core/mypy.ini +++ b/ofrak_core/mypy.ini @@ -16,7 +16,19 @@ ignore_missing_imports = True [mypy-xattr.*] ignore_missing_imports = True -[mypy-lief.*] +[mypy-lief.*] # TODO remove the next two lines when upgrading mypy. Current version has bug. +follow_imports = skip +follow_imports_for_stubs = true +ignore_missing_imports = True + +[mypy-tempfile312.*] # TODO remove the next two lines when upgrading mypy. Current version has bug. +follow_imports = skip +follow_imports_for_stubs = true +ignore_missing_imports = True + +[mypy-numpy.*] # TODO remove the next two lines when upgrading mypy. Current version has bug. +follow_imports = skip +follow_imports_for_stubs = true ignore_missing_imports = True [mypy-binwalk.*] diff --git a/ofrak_core/ofrak/ofrak_context.py b/ofrak_core/ofrak/ofrak_context.py index 8c0ee44d1..5bc54c91e 100644 --- a/ofrak_core/ofrak/ofrak_context.py +++ b/ofrak_core/ofrak/ofrak_context.py @@ -234,12 +234,12 @@ async def _get_discovered_components(self) -> List[ComponentInterface]: audited_components.append(component) else: components_missing_deps.append(component) - - LOGGER.warning( - f"Skipped registering the following components due to missing dependencies: " - f"{', '.join(type(c).__name__ for c in components_missing_deps)}. Run `python3 -m " - f"ofrak deps --missing-only` for more details." - ) + if len(components_missing_deps) > 0: + LOGGER.warning( + f"Skipped registering the following components due to missing dependencies: " + f"{', '.join(type(c).__name__ for c in components_missing_deps)}. Run `python3 -m " + f"ofrak deps --missing-only` for more details." + ) return audited_components diff --git a/ofrak_core/pytest_ofrak/elf/assets/Makefile b/ofrak_core/pytest_ofrak/elf/assets/Makefile index d1123333b..c568118ed 100644 --- a/ofrak_core/pytest_ofrak/elf/assets/Makefile +++ b/ofrak_core/pytest_ofrak/elf/assets/Makefile @@ -1,5 +1,5 @@ - -CC=gcc +# Specifying the full name is necessary for some tests to work on ARM +CC=x86_64-linux-gnu-gcc default: program diff --git a/ofrak_core/pytest_ofrak/elf/fixtures.py b/ofrak_core/pytest_ofrak/elf/fixtures.py index 99bbab873..ca97f6448 100644 --- a/ofrak_core/pytest_ofrak/elf/fixtures.py +++ b/ofrak_core/pytest_ofrak/elf/fixtures.py @@ -6,7 +6,7 @@ # MAKEFILE_CONTENTS = """ -# CC=gcc +# CC=x86_64-linux-gnu-gcc # default: program diff --git a/ofrak_core/requirements.txt b/ofrak_core/requirements.txt index 8fb088bae..f1d66e69f 100644 --- a/ofrak_core/requirements.txt +++ b/ofrak_core/requirements.txt @@ -10,8 +10,8 @@ importlib-metadata>=4.13 intervaltree==3.1.0 keystone-engine==0.9.2 jefferson==0.4.5;python_version>="3.8" -lief==0.15.1 -orjson~=3.9.15 +lief==0.16.1 +orjson~=3.10.12 pefile==2023.2.7 pycdlib==1.12.0 python-magic;platform_system!="Windows" diff --git a/ofrak_core/test_ofrak/service/serialization_service/test_pjson.py b/ofrak_core/test_ofrak/service/serialization_service/test_pjson.py index cb84aea41..bae6c4f97 100644 --- a/ofrak_core/test_ofrak/service/serialization_service/test_pjson.py +++ b/ofrak_core/test_ofrak/service/serialization_service/test_pjson.py @@ -76,11 +76,8 @@ def iterable_strategy(draw, type_hint): @composite def os_stat_result_strategy(draw, _type_hint): - """ - os.stat_result instances can be generated as tuples of size 10. They most likely won't be valid - but it doesn't matter here. - """ - return draw(tuples(*[integer_strategy()] * 10)) + raw_tuple = draw(tuples(*[integer_strategy()] * 10)) + return os.stat_result(raw_tuple) @composite diff --git a/ofrak_io/Makefile b/ofrak_io/Makefile index 0f5724824..cf7fde1c5 100644 --- a/ofrak_io/Makefile +++ b/ofrak_io/Makefile @@ -7,7 +7,7 @@ install: .PHONY: develop develop: - $(PIP) install -e .[test] + $(PIP) install -e .[test] --config-settings editable_mode=strict .PHONY: inspect inspect: @@ -16,4 +16,8 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_io --cov-report=term-missing --cov-fail-under=100 ofrak_io_test - fun-coverage --cov-fail-under=100 + @if [ "$(shell uname -m)" = "x86_64" ]; then \ + fun-coverage --cov-fail-under=100; \ + else \ + fun-coverage; \ + fi diff --git a/ofrak_io/ofrak_io/batch_manager.py b/ofrak_io/ofrak_io/batch_manager.py index ac743809b..09867062b 100644 --- a/ofrak_io/ofrak_io/batch_manager.py +++ b/ofrak_io/ofrak_io/batch_manager.py @@ -89,9 +89,10 @@ def __init__( async def get_result(self, request: Request) -> Result: current_batch = self._current_batch current_batch.add_request(request) + current_batch_task = asyncio.get_running_loop().create_task(current_batch.result(request)) # Gives self._handler_loop_task a chance to raise its errors done, _ = await asyncio.wait( - (current_batch.result(request), self._handler_loop_task), + (current_batch_task, self._handler_loop_task), return_when=asyncio.FIRST_COMPLETED, ) return next(iter(done)).result() diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index a157de7d9..be5b30b5b 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -22,13 +22,11 @@ RUN cd /tmp && \ #X64-64 toolchain for arm64 Docker images RUN if [ "$TARGETARCH" = "arm64" ]; then \ - apt-get update && apt-get install -y gcc-10-x86-64-linux-gnu; \ + apt-get update && apt-get install -y gcc-12-x86-64-linux-gnu; \ fi; #M68k GNU 10 Linux -#Only exists for x86 -RUN if [ "$TARGETARCH" = "amd64" ]; then \ - cd /tmp && \ +RUN cd /tmp && \ apt-get update && apt-get install -y gcc g++ gperf bison flex texinfo help2man make libncurses5-dev python3-dev autoconf automake libtool libtool-bin gawk wget bzip2 xz-utils unzip patch libstdc++6 rsync && \ git clone https://github.com/crosstool-ng/crosstool-ng.git && \ cd crosstool-ng/ && \ @@ -81,8 +79,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ 'CT_ZSTD_PATCH_GLOBAL=y' \ >> .config && \ ./ct-ng build CT_JOBS=`nproc` && \ - cd /tmp && rm -rf crosstool-ng; \ -fi; + cd /tmp && rm -rf crosstool-ng; #M68k VBCC RUN cd /tmp && \ @@ -108,6 +105,11 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ wget https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz -C /opt/rbs/toolchain && \ rm -rf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget https://developer.arm.com/-/media/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz --show-progress --progress=bar:force:noscroll && \ + tar xf gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz -C /opt/rbs/toolchain && \ + rm -rf gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz; \ fi; #AVR GCC @@ -116,6 +118,11 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ wget https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/SoftwareLibraries/Firmware/avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz --show-progress --progress=bar:force:noscroll && \ tar xzf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz -C /opt/rbs/toolchain && \ rm -rf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget http://downloads.arduino.cc/tools/avr-gcc-7.3.0-atmel3.6.1-arduino7-aarch64-pc-linux-gnu.tar.bz2 --show-progress --progress=bar:force:noscroll && \ + tar xf avr-gcc-7.3.0-atmel3.6.1-arduino7-aarch64-pc-linux-gnu.tar.bz2 -C /opt/rbs/toolchain && \ + rm -rf avr-gcc-7.3.0-atmel3.6.1-arduino7-aarch64-pc-linux-gnu.tar.bz2; \ fi; #PPC GNU 10 Linux @@ -124,6 +131,11 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ wget https://download.01.org/0day-ci/cross-package/gcc-10.3.0-nolibc/x86_64-gcc-10.3.0-nolibc_powerpc-linux.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf x86_64-gcc-10.3.0-nolibc_powerpc-linux.tar.xz -C /opt/rbs/toolchain && \ rm -rf x86_64-gcc-10.3.0-nolibc_powerpc-linux.tar.xz; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget https://www.kernel.org/pub/tools/crosstool/files/bin/arm64/10.3.0/arm64-gcc-10.3.0-nolibc-powerpc-linux.tar.xz --show-progress --progress=bar:force:noscroll && \ + tar xf arm64-gcc-10.3.0-nolibc-powerpc-linux.tar.xz -C /opt/rbs/toolchain && \ + rm -rf arm64-gcc-10.3.0-nolibc-powerpc-linux.tar.xz; \ fi; #BCC (GCC) SPARC v8 diff --git a/ofrak_patch_maker/Makefile b/ofrak_patch_maker/Makefile index 8c311007f..35856e4a4 100644 --- a/ofrak_patch_maker/Makefile +++ b/ofrak_patch_maker/Makefile @@ -8,7 +8,7 @@ install: .PHONY: develop develop: - $(PIP) install -e .[test] + $(PIP) install -e .[test] --config-settings editable_mode=strict .PHONY: inspect inspect: @@ -17,4 +17,8 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_patch_maker --cov-report=term-missing ofrak_patch_maker_test - fun-coverage --cov-fail-under=100 + @if [ "$(shell uname -m)" = "x86_64" ]; then \ + fun-coverage --cov-fail-under=100; \ + else \ + fun-coverage; \ + fi diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf index 769dab9b9..13578ce61 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf @@ -42,18 +42,18 @@ BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux- LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib [GNU_AARCH64_LINUX_10] -PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc -COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc -LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-objdump -LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/lib +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-objdump +LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/lib [GNU_AVR_5] -PREPROCESSOR = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc -COMPILER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc -LINKER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-ld -BIN_PARSER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-objdump -LIB = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/lib +PREPROCESSOR = /opt/rbs/toolchain/avr*/bin/avr-gcc +COMPILER = /opt/rbs/toolchain/avr*/bin/avr-gcc +LINKER = /opt/rbs/toolchain/avr*/bin/avr-ld +BIN_PARSER = /opt/rbs/toolchain/avr*/bin/avr-objdump +LIB = /opt/rbs/toolchain/avr*/lib [GNU_PPC_LINUX_10] PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc @@ -73,7 +73,7 @@ LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as M68K_ASM_PATH = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-as -AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-as -AVR_ASM_PATH = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-as +AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-as +AVR_ASM_PATH = /opt/rbs/toolchain/avr*/bin/avr-as PPC_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-as SPARC_ASM_PATH = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-as diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py index dbcdf0ce0..0d7faea73 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py @@ -1,5 +1,6 @@ import configparser import os +import glob from multiprocessing import Pool, cpu_count from typing import Optional, Dict, Mapping, Tuple @@ -28,6 +29,8 @@ def get_repository_config(section: str, key: Optional[str] = None): """ Get config values from toolchain.conf. + If the resulting value contains '*', we do a glob expansion. + :param section: section name in config file :param key: key in `config[section]` @@ -48,10 +51,18 @@ def get_repository_config(section: str, key: Optional[str] = None): try: config.read(conf) if key: - ret = config.get(section, key) + raw_value = config.get(section, key) + + if '*' in raw_value: + matches = glob.glob(raw_value) + if matches: + return matches[0] + else: + raise NotFoundError(f"No file matches wildcard {raw_value}") + else: + return raw_value else: - ret = config.items(section) # type: ignore - return ret + return config.items(section) # type: ignore except (configparser.NoSectionError, configparser.NoOptionError) as e: error_by_config_file[conf] = e continue diff --git a/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py b/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py index 3f2ebf178..f07a23082 100644 --- a/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py +++ b/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py @@ -1,4 +1,5 @@ import pytest +import platform from ofrak_patch_maker.toolchain.gnu_bcc_sparc import GNU_BCC_SPARC_Toolchain from ofrak_patch_maker_test import ToolchainUnderTest @@ -27,15 +28,16 @@ def toolchain_under_test(request) -> ToolchainUnderTest: return request.param - +@pytest.mark.skipif(platform.machine() != "x86_64", reason="Test only supported on x86_64") def test_monkey_patch(toolchain_under_test: ToolchainUnderTest): run_monkey_patch_test(toolchain_under_test) # C Tests +@pytest.mark.skipif(platform.machine() != "x86_64", reason="Test only supported on x86_64") def test_bounds_check(toolchain_under_test: ToolchainUnderTest): run_bounds_check_test(toolchain_under_test) - +@pytest.mark.skipif(platform.machine() != "x86_64", reason="Test only supported on x86_64") def test_hello_world(toolchain_under_test: ToolchainUnderTest): run_hello_world_test(toolchain_under_test) diff --git a/ofrak_tutorial/Makefile b/ofrak_tutorial/Makefile index f52592550..a6f0f0a91 100644 --- a/ofrak_tutorial/Makefile +++ b/ofrak_tutorial/Makefile @@ -29,7 +29,7 @@ run: .PHONY: develop develop: - ${PIP} install -e .[test] + ${PIP} install -e .[test] --config-settings editable_mode=strict make generate_stripped_notebooks .PHONY: install @@ -46,7 +46,11 @@ test: inspect $(PYTHON) -m pytest -n auto ofrak_tutorial_test # Note: this requires the Ghidra server to be up and running $(PYTHON) -m pytest --nbval --sanitize-with=nbval_sanitizer.cfg --cov=ofrak_tutorial --cov-report=term-missing --cov-fail-under=100 notebooks_with_outputs - fun-coverage --cov-fail-under=100 + @if [ "$(shell uname -m)" = "x86_64" ]; then \ + fun-coverage --cov-fail-under=100; \ + else \ + fun-coverage; \ + fi .PHONY: generate_stripped_notebooks generate_stripped_notebooks: diff --git a/ofrak_type/Makefile b/ofrak_type/Makefile index 2272aa437..d35cda760 100644 --- a/ofrak_type/Makefile +++ b/ofrak_type/Makefile @@ -7,7 +7,7 @@ install: .PHONY: install develop: - $(PIP) install -e .[test] + $(PIP) install -e .[test] --config-settings editable_mode=strict .PHONY: inspect inspect: @@ -16,4 +16,8 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_type --cov-report=term-missing --cov-fail-under=100 ofrak_type_test - fun-coverage --cov-fail-under=100 + @if [ "$(shell uname -m)" = "x86_64" ]; then \ + fun-coverage --cov-fail-under=100; \ + else \ + fun-coverage; \ + fi