From 34bb45712538381129723c10b2c84a6a5f06a832 Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 26 Dec 2024 14:38:26 -0700 Subject: [PATCH 01/28] upgrading angr as it uses the newer packages with prebuilt arm binaries --- disassemblers/ofrak_angr/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disassemblers/ofrak_angr/requirements.txt b/disassemblers/ofrak_angr/requirements.txt index 7bd0f95ba..3fb38294e 100644 --- a/disassemblers/ofrak_angr/requirements.txt +++ b/disassemblers/ofrak_angr/requirements.txt @@ -1,2 +1,2 @@ -angr==9.2.93 +angr==9.2.133 importlib-resources # A workaround for https://github.com/redballoonsecurity/ofrak/issues/398 From 33663ae1c09a722a10db6399f5ca5c5ef5770ec5 Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 26 Dec 2024 14:38:48 -0700 Subject: [PATCH 02/28] newer capstone with more arm pre-built binaries --- disassemblers/ofrak_capstone/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disassemblers/ofrak_capstone/requirements.txt b/disassemblers/ofrak_capstone/requirements.txt index 5903f94cb..22763a26e 100644 --- a/disassemblers/ofrak_capstone/requirements.txt +++ b/disassemblers/ofrak_capstone/requirements.txt @@ -1 +1 @@ -capstone==5.0.0.post1 +capstone==5.0.3 From ede642811d9becffdec01618bead91d662596064 Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 26 Dec 2024 14:42:24 -0700 Subject: [PATCH 03/28] gcc10 was phased out for gcc12 in bookworm --- ofrak_patch_maker/Dockerstub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index a157de7d9..6fe931e56 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -22,7 +22,7 @@ RUN cd /tmp && \ #X64-64 toolchain for arm64 Docker images RUN if [ "$TARGETARCH" = "arm64" ]; then \ - apt-get update && apt-get install -y gcc-10-x86-64-linux-gnu; \ + apt-get update && apt-get install -y gcc-12-x86-64-linux-gnu; \ fi; #M68k GNU 10 Linux From fca3fc070e25fc622b5d978a16c92c52ff4299f1 Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 26 Dec 2024 14:43:01 -0700 Subject: [PATCH 04/28] these packages needed newer versions to work with the newer python --- ofrak_core/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ofrak_core/requirements.txt b/ofrak_core/requirements.txt index 8fb088bae..2da7d6af4 100644 --- a/ofrak_core/requirements.txt +++ b/ofrak_core/requirements.txt @@ -10,8 +10,8 @@ importlib-metadata>=4.13 intervaltree==3.1.0 keystone-engine==0.9.2 jefferson==0.4.5;python_version>="3.8" -lief==0.15.1 -orjson~=3.9.15 +lief==0.16.0 +orjson~=3.10.12 pefile==2023.2.7 pycdlib==1.12.0 python-magic;platform_system!="Windows" From 4db5d2170bd9b32a7615a645426beaa873243c5b Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 2 Jan 2025 22:02:15 -0500 Subject: [PATCH 05/28] added multi arch versions as they allow x86 binary introspective tests to work on arm --- ofrak_core/Dockerstub | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ofrak_core/Dockerstub b/ofrak_core/Dockerstub index d2d76bfac..f340c3766 100644 --- a/ofrak_core/Dockerstub +++ b/ofrak_core/Dockerstub @@ -68,3 +68,17 @@ RUN cd /tmp && \ make install && \ cd /tmp && \ rm -r UEFITool-A68 + +# Multiarch packages necessary for various tests to work on ARM +RUN if [ "$TARGETARCH" = "arm64" ]; then \ + dpkg --add-architecture amd64 && apt-get update && \ + apt-get -y install --no-install-recommends \ + libc6:amd64 \ + zlib1g:amd64 \ + libselinux1:amd64 \ + libacl1:amd64 \ + libmpc3:amd64 \ + libisl23:amd64 \ + libstdc++6:amd64 \ + crossbuild-essential-amd64; \ +fi; \ No newline at end of file From 9463331021842232587e76d3a46b8129981b9d72 Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 2 Jan 2025 22:04:16 -0500 Subject: [PATCH 06/28] fixing a couple of small bugs and changing some small details to enable multiarch support --- ofrak_core/pytest_ofrak/elf/assets/Makefile | 4 ++-- ofrak_core/pytest_ofrak/elf/fixtures.py | 2 +- ofrak_patch_maker/Dockerstub | 22 ++++++++++++++++----- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/ofrak_core/pytest_ofrak/elf/assets/Makefile b/ofrak_core/pytest_ofrak/elf/assets/Makefile index d1123333b..c568118ed 100644 --- a/ofrak_core/pytest_ofrak/elf/assets/Makefile +++ b/ofrak_core/pytest_ofrak/elf/assets/Makefile @@ -1,5 +1,5 @@ - -CC=gcc +# Specifying the full name is necessary for some tests to work on ARM +CC=x86_64-linux-gnu-gcc default: program diff --git a/ofrak_core/pytest_ofrak/elf/fixtures.py b/ofrak_core/pytest_ofrak/elf/fixtures.py index 99bbab873..ca97f6448 100644 --- a/ofrak_core/pytest_ofrak/elf/fixtures.py +++ b/ofrak_core/pytest_ofrak/elf/fixtures.py @@ -6,7 +6,7 @@ # MAKEFILE_CONTENTS = """ -# CC=gcc +# CC=x86_64-linux-gnu-gcc # default: program diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index 6fe931e56..be5b30b5b 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -26,9 +26,7 @@ RUN if [ "$TARGETARCH" = "arm64" ]; then \ fi; #M68k GNU 10 Linux -#Only exists for x86 -RUN if [ "$TARGETARCH" = "amd64" ]; then \ - cd /tmp && \ +RUN cd /tmp && \ apt-get update && apt-get install -y gcc g++ gperf bison flex texinfo help2man make libncurses5-dev python3-dev autoconf automake libtool libtool-bin gawk wget bzip2 xz-utils unzip patch libstdc++6 rsync && \ git clone https://github.com/crosstool-ng/crosstool-ng.git && \ cd crosstool-ng/ && \ @@ -81,8 +79,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ 'CT_ZSTD_PATCH_GLOBAL=y' \ >> .config && \ ./ct-ng build CT_JOBS=`nproc` && \ - cd /tmp && rm -rf crosstool-ng; \ -fi; + cd /tmp && rm -rf crosstool-ng; #M68k VBCC RUN cd /tmp && \ @@ -108,6 +105,11 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ wget https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz -C /opt/rbs/toolchain && \ rm -rf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget https://developer.arm.com/-/media/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz --show-progress --progress=bar:force:noscroll && \ + tar xf gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz -C /opt/rbs/toolchain && \ + rm -rf gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz; \ fi; #AVR GCC @@ -116,6 +118,11 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ wget https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/SoftwareLibraries/Firmware/avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz --show-progress --progress=bar:force:noscroll && \ tar xzf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz -C /opt/rbs/toolchain && \ rm -rf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget http://downloads.arduino.cc/tools/avr-gcc-7.3.0-atmel3.6.1-arduino7-aarch64-pc-linux-gnu.tar.bz2 --show-progress --progress=bar:force:noscroll && \ + tar xf avr-gcc-7.3.0-atmel3.6.1-arduino7-aarch64-pc-linux-gnu.tar.bz2 -C /opt/rbs/toolchain && \ + rm -rf avr-gcc-7.3.0-atmel3.6.1-arduino7-aarch64-pc-linux-gnu.tar.bz2; \ fi; #PPC GNU 10 Linux @@ -124,6 +131,11 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ wget https://download.01.org/0day-ci/cross-package/gcc-10.3.0-nolibc/x86_64-gcc-10.3.0-nolibc_powerpc-linux.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf x86_64-gcc-10.3.0-nolibc_powerpc-linux.tar.xz -C /opt/rbs/toolchain && \ rm -rf x86_64-gcc-10.3.0-nolibc_powerpc-linux.tar.xz; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget https://www.kernel.org/pub/tools/crosstool/files/bin/arm64/10.3.0/arm64-gcc-10.3.0-nolibc-powerpc-linux.tar.xz --show-progress --progress=bar:force:noscroll && \ + tar xf arm64-gcc-10.3.0-nolibc-powerpc-linux.tar.xz -C /opt/rbs/toolchain && \ + rm -rf arm64-gcc-10.3.0-nolibc-powerpc-linux.tar.xz; \ fi; #BCC (GCC) SPARC v8 From a66c9b1d45d5c88a26be0c9c3a1544ee506bac87 Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 2 Jan 2025 22:05:06 -0500 Subject: [PATCH 07/28] I upgrade this and then they just released a new version --- ofrak_core/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_core/requirements.txt b/ofrak_core/requirements.txt index 2da7d6af4..f1d66e69f 100644 --- a/ofrak_core/requirements.txt +++ b/ofrak_core/requirements.txt @@ -10,7 +10,7 @@ importlib-metadata>=4.13 intervaltree==3.1.0 keystone-engine==0.9.2 jefferson==0.4.5;python_version>="3.8" -lief==0.16.0 +lief==0.16.1 orjson~=3.10.12 pefile==2023.2.7 pycdlib==1.12.0 From cd13de24bf74b4ee5d84f71ca3f3faa1590aafa3 Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 2 Jan 2025 22:41:39 -0500 Subject: [PATCH 08/28] Almost 100%...if the bcc toolchain had an arm version or build on arm, we'd have 100% test coverage on arm. Oh well, maybe a job for a different pr... --- ofrak_core/Makefile | 6 ++++- ofrak_io/Makefile | 6 ++++- ofrak_patch_maker/Makefile | 6 ++++- .../ofrak_patch_maker/toolchain.conf | 24 +++++++++---------- .../ofrak_patch_maker/toolchain/utils.py | 17 ++++++++++--- .../test_sparc_toolchain.py | 4 ++++ ofrak_tutorial/Makefile | 6 ++++- ofrak_type/Makefile | 6 ++++- 8 files changed, 55 insertions(+), 20 deletions(-) diff --git a/ofrak_core/Makefile b/ofrak_core/Makefile index c82f6be7e..0eec4f9fe 100644 --- a/ofrak_core/Makefile +++ b/ofrak_core/Makefile @@ -18,7 +18,11 @@ test: inspect $(PYTHON) -m pytest -n auto test_ofrak --cov=ofrak --cov-report=term-missing (sleep 2; echo 1; sleep 2; echo i agree) \ | python3 -m coverage run --append --source ofrak -m ofrak license --force - fun-coverage --cov-fail-under=100 + @if [ "$(shell uname -m)" = "x86_64" ]; then \ + fun-coverage --cov-fail-under=100; \ + else \ + fun-coverage; \ + fi ofrak/gui/public: if [ -d /ofrak_gui ] ; then \ diff --git a/ofrak_io/Makefile b/ofrak_io/Makefile index 0f5724824..a26009ba7 100644 --- a/ofrak_io/Makefile +++ b/ofrak_io/Makefile @@ -16,4 +16,8 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_io --cov-report=term-missing --cov-fail-under=100 ofrak_io_test - fun-coverage --cov-fail-under=100 + @if [ "$(shell uname -m)" = "x86_64" ]; then \ + fun-coverage --cov-fail-under=100; \ + else \ + fun-coverage; \ + fi diff --git a/ofrak_patch_maker/Makefile b/ofrak_patch_maker/Makefile index 8c311007f..23d0081c5 100644 --- a/ofrak_patch_maker/Makefile +++ b/ofrak_patch_maker/Makefile @@ -17,4 +17,8 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_patch_maker --cov-report=term-missing ofrak_patch_maker_test - fun-coverage --cov-fail-under=100 + @if [ "$(shell uname -m)" = "x86_64" ]; then \ + fun-coverage --cov-fail-under=100; \ + else \ + fun-coverage; \ + fi diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf index 769dab9b9..13578ce61 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf @@ -42,18 +42,18 @@ BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux- LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib [GNU_AARCH64_LINUX_10] -PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc -COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc -LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-objdump -LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/lib +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-objdump +LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/lib [GNU_AVR_5] -PREPROCESSOR = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc -COMPILER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc -LINKER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-ld -BIN_PARSER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-objdump -LIB = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/lib +PREPROCESSOR = /opt/rbs/toolchain/avr*/bin/avr-gcc +COMPILER = /opt/rbs/toolchain/avr*/bin/avr-gcc +LINKER = /opt/rbs/toolchain/avr*/bin/avr-ld +BIN_PARSER = /opt/rbs/toolchain/avr*/bin/avr-objdump +LIB = /opt/rbs/toolchain/avr*/lib [GNU_PPC_LINUX_10] PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc @@ -73,7 +73,7 @@ LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as M68K_ASM_PATH = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-as -AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-as -AVR_ASM_PATH = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-as +AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-as +AVR_ASM_PATH = /opt/rbs/toolchain/avr*/bin/avr-as PPC_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-as SPARC_ASM_PATH = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-as diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py index dbcdf0ce0..9273acb33 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py @@ -1,5 +1,6 @@ import configparser import os +import glob from multiprocessing import Pool, cpu_count from typing import Optional, Dict, Mapping, Tuple @@ -28,6 +29,8 @@ def get_repository_config(section: str, key: Optional[str] = None): """ Get config values from toolchain.conf. + If the resulting value contains '*', we do a glob expansion. + :param section: section name in config file :param key: key in `config[section]` @@ -48,10 +51,18 @@ def get_repository_config(section: str, key: Optional[str] = None): try: config.read(conf) if key: - ret = config.get(section, key) + raw_value = config.get(section, key) + + if "*" in raw_value: + matches = glob.glob(raw_value) + if matches: + return matches[0] + else: + raise NotFoundError(f"No file matches wildcard {raw_value}") + else: + return raw_value else: - ret = config.items(section) # type: ignore - return ret + return config.items(section) # type: ignore except (configparser.NoSectionError, configparser.NoOptionError) as e: error_by_config_file[conf] = e continue diff --git a/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py b/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py index 3f2ebf178..7886623c3 100644 --- a/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py +++ b/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py @@ -1,4 +1,5 @@ import pytest +import platform from ofrak_patch_maker.toolchain.gnu_bcc_sparc import GNU_BCC_SPARC_Toolchain from ofrak_patch_maker_test import ToolchainUnderTest @@ -28,14 +29,17 @@ def toolchain_under_test(request) -> ToolchainUnderTest: return request.param +@pytest.mark.skipif(platform.machine() != "x86_64", reason="Test only supported on x86_64") def test_monkey_patch(toolchain_under_test: ToolchainUnderTest): run_monkey_patch_test(toolchain_under_test) # C Tests +@pytest.mark.skipif(platform.machine() != "x86_64", reason="Test only supported on x86_64") def test_bounds_check(toolchain_under_test: ToolchainUnderTest): run_bounds_check_test(toolchain_under_test) +@pytest.mark.skipif(platform.machine() != "x86_64", reason="Test only supported on x86_64") def test_hello_world(toolchain_under_test: ToolchainUnderTest): run_hello_world_test(toolchain_under_test) diff --git a/ofrak_tutorial/Makefile b/ofrak_tutorial/Makefile index f52592550..bb9e6089e 100644 --- a/ofrak_tutorial/Makefile +++ b/ofrak_tutorial/Makefile @@ -46,7 +46,11 @@ test: inspect $(PYTHON) -m pytest -n auto ofrak_tutorial_test # Note: this requires the Ghidra server to be up and running $(PYTHON) -m pytest --nbval --sanitize-with=nbval_sanitizer.cfg --cov=ofrak_tutorial --cov-report=term-missing --cov-fail-under=100 notebooks_with_outputs - fun-coverage --cov-fail-under=100 + @if [ "$(shell uname -m)" = "x86_64" ]; then \ + fun-coverage --cov-fail-under=100; \ + else \ + fun-coverage; \ + fi .PHONY: generate_stripped_notebooks generate_stripped_notebooks: diff --git a/ofrak_type/Makefile b/ofrak_type/Makefile index 2272aa437..6466118b1 100644 --- a/ofrak_type/Makefile +++ b/ofrak_type/Makefile @@ -16,4 +16,8 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_type --cov-report=term-missing --cov-fail-under=100 ofrak_type_test - fun-coverage --cov-fail-under=100 + @if [ "$(shell uname -m)" = "x86_64" ]; then \ + fun-coverage --cov-fail-under=100; \ + else \ + fun-coverage; \ + fi From 49f792617aff62a5247ef39b60ed0ce7f1c43d62 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Tue, 7 Jan 2025 20:30:15 +0000 Subject: [PATCH 09/28] Fix missing newline in Dockerstub --- ofrak_core/Dockerstub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_core/Dockerstub b/ofrak_core/Dockerstub index f340c3766..ea8440ee2 100644 --- a/ofrak_core/Dockerstub +++ b/ofrak_core/Dockerstub @@ -81,4 +81,4 @@ RUN if [ "$TARGETARCH" = "arm64" ]; then \ libisl23:amd64 \ libstdc++6:amd64 \ crossbuild-essential-amd64; \ -fi; \ No newline at end of file +fi; From e74ee56a37dc9636561fbf54b2cdb3fa914527e3 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Thu, 9 Jan 2025 20:29:04 +0000 Subject: [PATCH 10/28] Update Makefiles with Jacob's suggestion --- disassemblers/ofrak_angr/Makefile | 2 +- disassemblers/ofrak_binary_ninja/Makefile | 2 +- disassemblers/ofrak_capstone/Makefile | 2 +- disassemblers/ofrak_ghidra/Makefile | 2 +- examples/Makefile | 2 +- ofrak_core/Makefile | 6 +----- ofrak_io/Makefile | 6 +----- ofrak_patch_maker/Makefile | 6 +----- ofrak_tutorial/Makefile | 6 +----- ofrak_type/Makefile | 6 +----- 10 files changed, 10 insertions(+), 30 deletions(-) diff --git a/disassemblers/ofrak_angr/Makefile b/disassemblers/ofrak_angr/Makefile index f7da7aa5e..99c1047e8 100644 --- a/disassemblers/ofrak_angr/Makefile +++ b/disassemblers/ofrak_angr/Makefile @@ -16,4 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_angr ofrak_angr_test - fun-coverage --cov-fail-under=100 + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) diff --git a/disassemblers/ofrak_binary_ninja/Makefile b/disassemblers/ofrak_binary_ninja/Makefile index dd2052fa1..53f437956 100644 --- a/disassemblers/ofrak_binary_ninja/Makefile +++ b/disassemblers/ofrak_binary_ninja/Makefile @@ -16,4 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest --cov=ofrak_binary_ninja --cov-report=term-missing ofrak_binary_ninja_test - fun-coverage --cov-fail-under=100 + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) diff --git a/disassemblers/ofrak_capstone/Makefile b/disassemblers/ofrak_capstone/Makefile index 71746f688..af002f21d 100644 --- a/disassemblers/ofrak_capstone/Makefile +++ b/disassemblers/ofrak_capstone/Makefile @@ -16,4 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest ofrak_capstone_test --cov=ofrak_capstone --cov-report=term-missing - fun-coverage --cov-fail-under=100 + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) diff --git a/disassemblers/ofrak_ghidra/Makefile b/disassemblers/ofrak_ghidra/Makefile index c4a4f96bd..7f0f129d1 100644 --- a/disassemblers/ofrak_ghidra/Makefile +++ b/disassemblers/ofrak_ghidra/Makefile @@ -9,4 +9,4 @@ develop: test: $(PYTHON) -m pytest --cov=ofrak_ghidra --cov-report=term-missing ofrak_ghidra_test - fun-coverage --cov-fail-under=100 + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) diff --git a/examples/Makefile b/examples/Makefile index e4f917bf4..ac8a224d0 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -10,7 +10,7 @@ develop: .PHONY: test test: $(PYTHON) -m pytest --cov=. --cov-report=term-missing test_examples.py - fun-coverage --cov-fail-under=100 + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) .PHONY: dependencies dependencies: diff --git a/ofrak_core/Makefile b/ofrak_core/Makefile index 0eec4f9fe..9759e0d26 100644 --- a/ofrak_core/Makefile +++ b/ofrak_core/Makefile @@ -18,11 +18,7 @@ test: inspect $(PYTHON) -m pytest -n auto test_ofrak --cov=ofrak --cov-report=term-missing (sleep 2; echo 1; sleep 2; echo i agree) \ | python3 -m coverage run --append --source ofrak -m ofrak license --force - @if [ "$(shell uname -m)" = "x86_64" ]; then \ - fun-coverage --cov-fail-under=100; \ - else \ - fun-coverage; \ - fi + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) ofrak/gui/public: if [ -d /ofrak_gui ] ; then \ diff --git a/ofrak_io/Makefile b/ofrak_io/Makefile index a26009ba7..f4b326861 100644 --- a/ofrak_io/Makefile +++ b/ofrak_io/Makefile @@ -16,8 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_io --cov-report=term-missing --cov-fail-under=100 ofrak_io_test - @if [ "$(shell uname -m)" = "x86_64" ]; then \ - fun-coverage --cov-fail-under=100; \ - else \ - fun-coverage; \ - fi + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) diff --git a/ofrak_patch_maker/Makefile b/ofrak_patch_maker/Makefile index 23d0081c5..7c6d72f17 100644 --- a/ofrak_patch_maker/Makefile +++ b/ofrak_patch_maker/Makefile @@ -17,8 +17,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_patch_maker --cov-report=term-missing ofrak_patch_maker_test - @if [ "$(shell uname -m)" = "x86_64" ]; then \ - fun-coverage --cov-fail-under=100; \ - else \ - fun-coverage; \ - fi + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) diff --git a/ofrak_tutorial/Makefile b/ofrak_tutorial/Makefile index bb9e6089e..e6626b7ca 100644 --- a/ofrak_tutorial/Makefile +++ b/ofrak_tutorial/Makefile @@ -46,11 +46,7 @@ test: inspect $(PYTHON) -m pytest -n auto ofrak_tutorial_test # Note: this requires the Ghidra server to be up and running $(PYTHON) -m pytest --nbval --sanitize-with=nbval_sanitizer.cfg --cov=ofrak_tutorial --cov-report=term-missing --cov-fail-under=100 notebooks_with_outputs - @if [ "$(shell uname -m)" = "x86_64" ]; then \ - fun-coverage --cov-fail-under=100; \ - else \ - fun-coverage; \ - fi + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) .PHONY: generate_stripped_notebooks generate_stripped_notebooks: diff --git a/ofrak_type/Makefile b/ofrak_type/Makefile index 6466118b1..4ddc0fadc 100644 --- a/ofrak_type/Makefile +++ b/ofrak_type/Makefile @@ -16,8 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_type --cov-report=term-missing --cov-fail-under=100 ofrak_type_test - @if [ "$(shell uname -m)" = "x86_64" ]; then \ - fun-coverage --cov-fail-under=100; \ - else \ - fun-coverage; \ - fi + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) From a78622bfba3e85e96bcd5428d085404c7aa93d67 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Thu, 9 Jan 2025 20:42:11 +0000 Subject: [PATCH 11/28] Add >= version specifier to angr requirement for different python versions --- disassemblers/ofrak_angr/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disassemblers/ofrak_angr/requirements.txt b/disassemblers/ofrak_angr/requirements.txt index 3fb38294e..ae657b7fc 100644 --- a/disassemblers/ofrak_angr/requirements.txt +++ b/disassemblers/ofrak_angr/requirements.txt @@ -1,2 +1,2 @@ -angr==9.2.133 +angr>=9.2.93 importlib-resources # A workaround for https://github.com/redballoonsecurity/ofrak/issues/398 From 0b0a23f6612a105c8ad5be0646aa5ec91392541c Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Thu, 9 Jan 2025 20:46:38 +0000 Subject: [PATCH 12/28] remove incorrect "shell" in subshell --- ofrak_type/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_type/Makefile b/ofrak_type/Makefile index 4ddc0fadc..18c6ad59c 100644 --- a/ofrak_type/Makefile +++ b/ofrak_type/Makefile @@ -16,4 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_type --cov-report=term-missing --cov-fail-under=100 ofrak_type_test - fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) + fun-coverage --cov-fail-under=100 || ! (uname -m | grep x86_64) From 99518815592ab1d446a1056dd916817e385c0576 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Fri, 17 Jan 2025 16:30:33 +0000 Subject: [PATCH 13/28] Add separate configuration files for amd64 and arm64 docker images --- .../conf/toolchain_docker_amd64.conf | 79 +++++++++++++++++++ .../conf/toolchain_docker_arm64.conf | 79 +++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 ofrak_patch_maker/conf/toolchain_docker_amd64.conf create mode 100644 ofrak_patch_maker/conf/toolchain_docker_arm64.conf diff --git a/ofrak_patch_maker/conf/toolchain_docker_amd64.conf b/ofrak_patch_maker/conf/toolchain_docker_amd64.conf new file mode 100644 index 000000000..769dab9b9 --- /dev/null +++ b/ofrak_patch_maker/conf/toolchain_docker_amd64.conf @@ -0,0 +1,79 @@ +[LLVM_12_0_1] +PREPROCESSOR = /opt/rbs/toolchain/llvm_12.0.1/bin/clang +COMPILER = /opt/rbs/toolchain/llvm_12.0.1/bin/clang +LINKER = /opt/rbs/toolchain/llvm_12.0.1/bin/ld.lld +BIN_PARSER = /opt/rbs/toolchain/llvm_12.0.1/bin/llvm-readobj +LIB = /opt/rbs/toolchain/llvm_12.0.1/lib + +[GNU_ARM_NONE_EABI_10_2_1] +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-objdump +LIB = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/lib + +[GNU_M68K_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump +LIB=/opt/rbs/toolchain/m68k-unknown-linux-gnu/lib + +[VBCC_M68K_0_9] +PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/vbcc_0_9/bin/vbccm68k +LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump + +[GNU_X86_64_LINUX_EABI_10_3_0] +# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib + +[GNU_X86_32_LINUX_EABI_10_3_0] +# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib + +[GNU_AARCH64_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-objdump +LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/lib + +[GNU_AVR_5] +PREPROCESSOR = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc +COMPILER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc +LINKER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-ld +BIN_PARSER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-objdump +LIB = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/lib + +[GNU_PPC_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/lib + +[BCC_SPARC_GAISLER_ELF] +PREPROCESSOR = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc +COMPILER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc +LINKER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-ld +BIN_PARSER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-objdump +LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib + +[ASM] +ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as +X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as +M68K_ASM_PATH = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-as +AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-as +AVR_ASM_PATH = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-as +PPC_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-as +SPARC_ASM_PATH = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-as diff --git a/ofrak_patch_maker/conf/toolchain_docker_arm64.conf b/ofrak_patch_maker/conf/toolchain_docker_arm64.conf new file mode 100644 index 000000000..935f46f2b --- /dev/null +++ b/ofrak_patch_maker/conf/toolchain_docker_arm64.conf @@ -0,0 +1,79 @@ +[LLVM_12_0_1] +PREPROCESSOR = /opt/rbs/toolchain/llvm_12.0.1/bin/clang +COMPILER = /opt/rbs/toolchain/llvm_12.0.1/bin/clang +LINKER = /opt/rbs/toolchain/llvm_12.0.1/bin/ld.lld +BIN_PARSER = /opt/rbs/toolchain/llvm_12.0.1/bin/llvm-readobj +LIB = /opt/rbs/toolchain/llvm_12.0.1/lib + +[GNU_ARM_NONE_EABI_10_2_1] +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-objdump +LIB = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/lib + +[GNU_M68K_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump +LIB=/opt/rbs/toolchain/m68k-unknown-linux-gnu/lib + +[VBCC_M68K_0_9] +PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/vbcc_0_9/bin/vbccm68k +LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump + +[GNU_X86_64_LINUX_EABI_10_3_0] +# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib + +[GNU_X86_32_LINUX_EABI_10_3_0] +# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib + +[GNU_AARCH64_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-objdump +LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/lib + +[GNU_AVR_5] +PREPROCESSOR = /opt/rbs/toolchain/avr/bin/avr-gcc +COMPILER = /opt/rbs/toolchain/avr/bin/avr-gcc +LINKER = /opt/rbs/toolchain/avr/bin/avr-ld +BIN_PARSER = /opt/rbs/toolchain/avr/bin/avr-objdump +LIB = /opt/rbs/toolchain/avr/lib + +[GNU_PPC_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/lib + +[BCC_SPARC_GAISLER_ELF] +PREPROCESSOR = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc +COMPILER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc +LINKER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-ld +BIN_PARSER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-objdump +LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib + +[ASM] +ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as +X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as +M68K_ASM_PATH = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-as +AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-as +AVR_ASM_PATH = /opt/rbs/toolchain/avr/bin/avr-as +PPC_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-as +SPARC_ASM_PATH = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-as From 35c79b478e5554375d8f88fef564ad5add9a8d41 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Fri, 17 Jan 2025 16:42:44 +0000 Subject: [PATCH 14/28] Look in /etc/ofrak for toolchain.conf --- .../ofrak_patch_maker/toolchain/abstract.py | 12 ++-- .../ofrak_patch_maker/toolchain/utils.py | 67 +++++++++++++------ 2 files changed, 52 insertions(+), 27 deletions(-) diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py index 0b53fc698..d13fe125d 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py @@ -12,7 +12,7 @@ from ofrak_type import ArchInfo from ofrak_patch_maker.binary_parser.abstract import AbstractBinaryFileParser from ofrak_patch_maker.toolchain.model import Segment, ToolchainConfig -from ofrak_patch_maker.toolchain.utils import get_repository_config +from ofrak_patch_maker.toolchain.utils import get_exec_from_config, get_repository_config from ofrak_type.architecture import InstructionSet from ofrak_type.bit_width import BitWidth from ofrak_type.memory_permissions import MemoryPermissions @@ -157,35 +157,35 @@ def _assembler_path(self) -> str: assembler_path = "SPARC_ASM_PATH" else: assembler_path = f"{self._processor.isa.value.upper()}_ASM_PATH" - return get_repository_config("ASM", assembler_path) + return get_exec_from_config("ASM", assembler_path) @property def _preprocessor_path(self) -> str: """ :return str: path to the toolchain preprocessor - this is usually the compiler. """ - return get_repository_config(self.name, "PREPROCESSOR") + return get_exec_from_config(self.name, "PREPROCESSOR") @property def _compiler_path(self) -> str: """ :return str: path to the toolchain compiler """ - return get_repository_config(self.name, "COMPILER") + return get_exec_from_config(self.name, "COMPILER") @property def _linker_path(self) -> str: """ :return str: path to the toolchain linker """ - return get_repository_config(self.name, "LINKER") + return get_exec_from_config(self.name, "LINKER") @property def _readobj_path(self) -> str: """ :return str: path to the toolchain binary analysis utility """ - return get_repository_config(self.name, "BIN_PARSER") + return get_exec_from_config(self.name, "BIN_PARSER") @property def _lib_path(self) -> str: diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py index 9273acb33..cd597acdf 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py @@ -1,10 +1,9 @@ import configparser import os -import glob from multiprocessing import Pool, cpu_count from typing import Optional, Dict, Mapping, Tuple - import math +import shutil from ofrak_patch_maker.toolchain.model import BinFileType, Segment from ofrak_type.error import NotFoundError @@ -27,9 +26,8 @@ def get_file_format(path): def get_repository_config(section: str, key: Optional[str] = None): """ - Get config values from toolchain.conf. - - If the resulting value contains '*', we do a glob expansion. + Get config values from toolchain.conf. The toolchain.conf file can be located /etc/ofrak/toolchain.conf + or inside the `ofrak_patch_maker` Python package install. :param section: section name in config file :param key: key in `config[section]` @@ -41,34 +39,33 @@ def get_repository_config(section: str, key: Optional[str] = None): config = configparser.RawConfigParser() config_name = "toolchain.conf" - local_config = os.path.join(os.path.dirname(__file__), os.path.pardir) - config_paths = [local_config] + + # Should be shared across main OFRAK and use standardized configuration dirs in the future + config_paths = [ + os.path.join(p, config_name) + for p in ("/etc/ofrak/", os.path.join(os.path.dirname(__file__), os.path.pardir)) + ] + error_by_config_file: Dict[str, Exception] = dict() - for p in config_paths: - conf = os.path.join(p, config_name) + for conf in config_paths: if not os.path.exists(conf): continue try: config.read(conf) if key: - raw_value = config.get(section, key) - - if "*" in raw_value: - matches = glob.glob(raw_value) - if matches: - return matches[0] - else: - raise NotFoundError(f"No file matches wildcard {raw_value}") - else: - return raw_value + ret = config.get(section, key) else: - return config.items(section) # type: ignore + ret = config.items(section) # type: ignore + return ret except (configparser.NoSectionError, configparser.NoOptionError) as e: error_by_config_file[conf] = e continue if 0 == len(error_by_config_file): - raise NotFoundError(f"Configuration file {config_name} not found") + config_paths_str = "\n".join(config_paths) + raise NotFoundError( + f"Configuration file {config_name} not found. Tried the following locations:\n{config_paths_str}" + ) elif 1 == len(error_by_config_file): _config, _e = next(iter(error_by_config_file.items())) @@ -81,6 +78,34 @@ def get_repository_config(section: str, key: Optional[str] = None): ) +def get_exec_from_config(section: str, key: str): + """ + Get an executable name or path from a toolchain.conf. Output instructions for configuring + toolchain paths if the exectuable is not available. + + :param section: section name in config file + :param key: key in `config[section]` + + :raises SystemExit: If `config[section]` or `config[section][key]` not found. + :return Union[str, List[Tuple[str, str]]]: the result of ``config.get(section, key)`` or + ``config.items(section)`` + """ + + exec_path = get_repository_config(section, key) + if shutil.which(exec_path) is not None: + return exec_path + else: + try: + download_link = get_repository_config(section, "DOWNLOAD") + download_text = f"download the toolchain from {download_link} and " + except NotFoundError: + download_text = "" + raise NotFoundError( + f"Configured executable {exec_path} ({key} of {section}) not found on the filesystem or PATH. " + f"Please {download_text}edit the toolchain.conf file to point to the toolchain executable." + ) + + # TODO: Add a main driver for this guy NULL_DATA = Segment( segment_name=".data", From aa70d01a5dd08027cbebf9f2a1ccc77d3940393c Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Fri, 17 Jan 2025 16:57:01 +0000 Subject: [PATCH 15/28] Copy platform specific Docker toolchain.conf to /etc/ofrak/toolchain.conf --- ofrak_patch_maker/Dockerstub | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index be5b30b5b..df4fa8bf8 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -1,5 +1,7 @@ ARG TARGETARCH +COPY ${PACKAGE_PATH}/conf/toolchain_docker_${TARGETARCH}.conf /etc/ofrak/toolchain.conf + # LLVM RUN mkdir -p /opt/rbs/toolchain && \ cd /tmp && \ From 82c7a544317dc1a32cefe7ab7c9e8c8ebed82b0d Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Fri, 17 Jan 2025 18:50:45 +0000 Subject: [PATCH 16/28] Revert toolchain.conf to master to avoid breaking existing containers --- .../ofrak_patch_maker/toolchain.conf | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf index 13578ce61..769dab9b9 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf @@ -42,18 +42,18 @@ BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux- LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib [GNU_AARCH64_LINUX_10] -PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-gcc -COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-gcc -LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-objdump -LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/lib +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-objdump +LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/lib [GNU_AVR_5] -PREPROCESSOR = /opt/rbs/toolchain/avr*/bin/avr-gcc -COMPILER = /opt/rbs/toolchain/avr*/bin/avr-gcc -LINKER = /opt/rbs/toolchain/avr*/bin/avr-ld -BIN_PARSER = /opt/rbs/toolchain/avr*/bin/avr-objdump -LIB = /opt/rbs/toolchain/avr*/lib +PREPROCESSOR = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc +COMPILER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc +LINKER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-ld +BIN_PARSER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-objdump +LIB = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/lib [GNU_PPC_LINUX_10] PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc @@ -73,7 +73,7 @@ LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as M68K_ASM_PATH = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-as -AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-as -AVR_ASM_PATH = /opt/rbs/toolchain/avr*/bin/avr-as +AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-as +AVR_ASM_PATH = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-as PPC_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-as SPARC_ASM_PATH = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-as From d0020b9b1872dcf219300a90e349ac2497e2d983 Mon Sep 17 00:00:00 2001 From: Wyatt <53830972+whyitfor@users.noreply.github.com> Date: Mon, 14 Apr 2025 18:08:41 -0400 Subject: [PATCH 17/28] Revert changes to functional coverage --- disassemblers/ofrak_angr/Makefile | 2 +- disassemblers/ofrak_binary_ninja/Makefile | 2 +- disassemblers/ofrak_capstone/Makefile | 2 +- disassemblers/ofrak_ghidra/Makefile | 2 +- examples/Makefile | 2 +- ofrak_core/Makefile | 2 +- ofrak_io/Makefile | 2 +- ofrak_patch_maker/Makefile | 2 +- ofrak_tutorial/Makefile | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/disassemblers/ofrak_angr/Makefile b/disassemblers/ofrak_angr/Makefile index b91be2620..fb8cc4b83 100644 --- a/disassemblers/ofrak_angr/Makefile +++ b/disassemblers/ofrak_angr/Makefile @@ -16,4 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_angr ofrak_angr_test - fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) + fun-coverage --cov-fail-under=100 diff --git a/disassemblers/ofrak_binary_ninja/Makefile b/disassemblers/ofrak_binary_ninja/Makefile index ab082acdf..e9c3f30a7 100644 --- a/disassemblers/ofrak_binary_ninja/Makefile +++ b/disassemblers/ofrak_binary_ninja/Makefile @@ -16,4 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest --cov=ofrak_binary_ninja --cov-report=term-missing ofrak_binary_ninja_test - fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) + fun-coverage --cov-fail-under=100 diff --git a/disassemblers/ofrak_capstone/Makefile b/disassemblers/ofrak_capstone/Makefile index 552592683..88e4cfe32 100644 --- a/disassemblers/ofrak_capstone/Makefile +++ b/disassemblers/ofrak_capstone/Makefile @@ -16,4 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest ofrak_capstone_test --cov=ofrak_capstone --cov-report=term-missing - fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) + fun-coverage --cov-fail-under=100 diff --git a/disassemblers/ofrak_ghidra/Makefile b/disassemblers/ofrak_ghidra/Makefile index 557efc0a1..dc78fcc47 100644 --- a/disassemblers/ofrak_ghidra/Makefile +++ b/disassemblers/ofrak_ghidra/Makefile @@ -9,4 +9,4 @@ develop: test: $(PYTHON) -m pytest --cov=ofrak_ghidra --cov-report=term-missing ofrak_ghidra_test - fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) + fun-coverage --cov-fail-under=100 diff --git a/examples/Makefile b/examples/Makefile index ac8a224d0..e4f917bf4 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -10,7 +10,7 @@ develop: .PHONY: test test: $(PYTHON) -m pytest --cov=. --cov-report=term-missing test_examples.py - fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) + fun-coverage --cov-fail-under=100 .PHONY: dependencies dependencies: diff --git a/ofrak_core/Makefile b/ofrak_core/Makefile index fa5167882..847b3cdd3 100644 --- a/ofrak_core/Makefile +++ b/ofrak_core/Makefile @@ -18,7 +18,7 @@ test: inspect $(PYTHON) -m pytest -n auto test_ofrak --cov=ofrak --cov-report=term-missing (sleep 2; echo 1; sleep 2; echo i agree) \ | python3 -m coverage run --append --source ofrak -m ofrak license --force - fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) + fun-coverage --cov-fail-under=100 ofrak/gui/public: if [ -d /ofrak_gui ] ; then \ diff --git a/ofrak_io/Makefile b/ofrak_io/Makefile index d9938f1d2..f49e49397 100644 --- a/ofrak_io/Makefile +++ b/ofrak_io/Makefile @@ -16,4 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_io --cov-report=term-missing --cov-fail-under=100 ofrak_io_test - fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) + fun-coverage --cov-fail-under=100 diff --git a/ofrak_patch_maker/Makefile b/ofrak_patch_maker/Makefile index eb2f27f8e..5466c18f7 100644 --- a/ofrak_patch_maker/Makefile +++ b/ofrak_patch_maker/Makefile @@ -17,4 +17,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_patch_maker --cov-report=term-missing ofrak_patch_maker_test - fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) + fun-coverage --cov-fail-under=100 diff --git a/ofrak_tutorial/Makefile b/ofrak_tutorial/Makefile index 27d61d15d..d1e70c727 100644 --- a/ofrak_tutorial/Makefile +++ b/ofrak_tutorial/Makefile @@ -46,7 +46,7 @@ test: inspect $(PYTHON) -m pytest -n auto ofrak_tutorial_test # Note: this requires the Ghidra server to be up and running $(PYTHON) -m pytest --nbval --sanitize-with=nbval_sanitizer.cfg --cov=ofrak_tutorial --cov-report=term-missing --cov-fail-under=100 notebooks_with_outputs - fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) + fun-coverage --cov-fail-under=100 .PHONY: generate_stripped_notebooks generate_stripped_notebooks: From 711b9279d2062e27036f8bf95e8808f1ec44e12b Mon Sep 17 00:00:00 2001 From: Wyatt <53830972+whyitfor@users.noreply.github.com> Date: Mon, 14 Apr 2025 20:52:39 -0400 Subject: [PATCH 18/28] Revert M68K changes --- ofrak_patch_maker/Dockerstub | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index 8d84cec80..482ff66df 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -28,7 +28,9 @@ RUN if [ "$TARGETARCH" = "arm64" ]; then \ fi; #M68k GNU 10 Linux -RUN cd /tmp && \ +#Only exists for x86 +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + cd /tmp && \ apt-get update && apt-get install -y gcc g++ gperf bison flex texinfo help2man make libncurses5-dev python3-dev autoconf automake libtool libtool-bin gawk wget bzip2 xz-utils unzip patch libstdc++6 rsync && \ git clone https://github.com/crosstool-ng/crosstool-ng.git && \ cd crosstool-ng/ && \ @@ -100,7 +102,7 @@ RUN cd /tmp && \ CPU=m68k SYNTAX=mot make && \ cp ./vasmm68k_mot /opt/rbs/toolchain/vbcc_0_9/bin/ && \ cp ./vobjdump /opt/rbs/toolchain/vbcc_0_9/bin/ && \ - cd /tmp &&\ + cd /tmp && \ rm -rf vasm* vbcc* #AARCH64 GNU 10 Linux From 4e4bfee1c3fe00dd975ab199a462bf973035c2fe Mon Sep 17 00:00:00 2001 From: Wyatt <53830972+whyitfor@users.noreply.github.com> Date: Mon, 14 Apr 2025 21:03:19 -0400 Subject: [PATCH 19/28] Address MR feedback --- disassemblers/ofrak_angr/CHANGELOG.md | 2 +- disassemblers/ofrak_angr/requirements.txt | 2 +- ofrak_core/pytest_ofrak/elf/assets/Makefile | 3 +- ofrak_core/pytest_ofrak/elf/fixtures.py | 121 -------------------- 4 files changed, 3 insertions(+), 125 deletions(-) diff --git a/disassemblers/ofrak_angr/CHANGELOG.md b/disassemblers/ofrak_angr/CHANGELOG.md index 061a5550b..16259123b 100644 --- a/disassemblers/ofrak_angr/CHANGELOG.md +++ b/disassemblers/ofrak_angr/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) ### Changed -- Update to latest angr==9.2.77, which also necessitates Python >= 3.8. +- Update to latest angr==9.2.93, which also necessitates Python >= 3.8. - Refactored AngrDecompilationAnalysis/Analyzer to use generic components in ofrak core. ([#453](https://github.com/redballoonsecurity/ofrak/pull/453)) - Minor update to OFRAK Community License, add OFRAK Pro License ([#478](https://github.com/redballoonsecurity/ofrak/pull/478)) diff --git a/disassemblers/ofrak_angr/requirements.txt b/disassemblers/ofrak_angr/requirements.txt index ae657b7fc..7bd0f95ba 100644 --- a/disassemblers/ofrak_angr/requirements.txt +++ b/disassemblers/ofrak_angr/requirements.txt @@ -1,2 +1,2 @@ -angr>=9.2.93 +angr==9.2.93 importlib-resources # A workaround for https://github.com/redballoonsecurity/ofrak/issues/398 diff --git a/ofrak_core/pytest_ofrak/elf/assets/Makefile b/ofrak_core/pytest_ofrak/elf/assets/Makefile index c568118ed..121d60e88 100644 --- a/ofrak_core/pytest_ofrak/elf/assets/Makefile +++ b/ofrak_core/pytest_ofrak/elf/assets/Makefile @@ -1,5 +1,4 @@ -# Specifying the full name is necessary for some tests to work on ARM -CC=x86_64-linux-gnu-gcc +CC=gcc default: program diff --git a/ofrak_core/pytest_ofrak/elf/fixtures.py b/ofrak_core/pytest_ofrak/elf/fixtures.py index ca97f6448..32eedfeeb 100644 --- a/ofrak_core/pytest_ofrak/elf/fixtures.py +++ b/ofrak_core/pytest_ofrak/elf/fixtures.py @@ -1,133 +1,12 @@ import os -# import subprocess - import pytest -# MAKEFILE_CONTENTS = """ -# CC=x86_64-linux-gnu-gcc - -# default: program - -# program.o: program.c $(HEADERS) -# $(CC) -c program.c -fno-asynchronous-unwind-tables -o program.o - -# program: program.o -# $(CC) program.o -o program - -# program_no_reloc: program.o -# $(CC) program.o -no-pie -o program_no_reloc - -# program_relocated: program_relocated.o -# $(CC) program_relocated.o -o program_relocated - -# large_elf.o: large_elf.c $(HEADERS) -# $(CC) -c large_elf.c -o large_elf.o - -# large_elf: large_elf.o -# $(CC) large_elf.o -no-pie -o large_elf -# """ - -# C_SOURCE_CONTENTS = """ -# #include - -# int foo(); -# int bar(); - -# int main() { -# printf("Hello, World!\\n"); -# return foo(); -# } - -# int foo() { -# return 12; -# } - -# int bar() { -# return 24; -# } - - -# """ - -# LARGE_SOURCE_CONTENTS_HEADER = """ -# int foo(); -# int bar(); - -# int main() { -# return bar(); -# } - -# """ - -# LARGE_SOURCE_CONTENTS_FOOTER = """ - -# int foo() { -# return 12; -# } - -# int bar() { -# return 24; -# } - - -# """ - -# PATCH_CONTENTS = """ -# int noop0(); - -# int baz() -# { -# noop0(); -# return 36; -# } -# """ - - -# def create_noops(): -# noops = [] - -# i = 0 - -# for i in range(8000): -# instruction = f"int noop{i}(){{}}" -# noops.append(instruction) - -# return noops - - @pytest.fixture(scope="session") def elf_test_directory(): return os.path.join(os.path.dirname(__file__), "assets") - # makefile_path = os.path.join(tmpdir, "Makefile") - # c_source_path = os.path.join(tmpdir, "program.c") - # large_source_path = os.path.join(tmpdir, "large_elf.c") - - # patch_dir = os.path.join(tmpdir, "source_dir") - # if not os.path.exists(patch_dir): - # os.mkdir(patch_dir) - # patch_path = os.path.join(patch_dir, "patch.c") - - # noops = create_noops() - - # LARGE_SOURCE_CONTENTS = LARGE_SOURCE_CONTENTS_HEADER - # for noop in noops: - # LARGE_SOURCE_CONTENTS += noop - # LARGE_SOURCE_CONTENTS += LARGE_SOURCE_CONTENTS_FOOTER - - # with open(makefile_path, "w") as f: - # f.write(MAKEFILE_CONTENTS) - # with open(c_source_path, "w") as f: - # f.write(C_SOURCE_CONTENTS) - # with open(large_source_path, "w") as f: - # f.write(LARGE_SOURCE_CONTENTS) - # with open(patch_path, "w") as f: - # f.write(PATCH_CONTENTS) - - # return tmpdir - @pytest.fixture(scope="session") def elf_object_file(elf_test_directory): From cd4ca9d096c3291ddf309b55d45e5a10b9705e8d Mon Sep 17 00:00:00 2001 From: Wyatt <53830972+whyitfor@users.noreply.github.com> Date: Tue, 15 Apr 2025 11:47:25 -0400 Subject: [PATCH 20/28] Remove unneded changes --- ofrak_core/Dockerstub | 14 -------------- ofrak_type/Makefile | 2 +- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/ofrak_core/Dockerstub b/ofrak_core/Dockerstub index ea8440ee2..d2d76bfac 100644 --- a/ofrak_core/Dockerstub +++ b/ofrak_core/Dockerstub @@ -68,17 +68,3 @@ RUN cd /tmp && \ make install && \ cd /tmp && \ rm -r UEFITool-A68 - -# Multiarch packages necessary for various tests to work on ARM -RUN if [ "$TARGETARCH" = "arm64" ]; then \ - dpkg --add-architecture amd64 && apt-get update && \ - apt-get -y install --no-install-recommends \ - libc6:amd64 \ - zlib1g:amd64 \ - libselinux1:amd64 \ - libacl1:amd64 \ - libmpc3:amd64 \ - libisl23:amd64 \ - libstdc++6:amd64 \ - crossbuild-essential-amd64; \ -fi; diff --git a/ofrak_type/Makefile b/ofrak_type/Makefile index 5e6b041bf..f448930ce 100644 --- a/ofrak_type/Makefile +++ b/ofrak_type/Makefile @@ -16,4 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_type --cov-report=term-missing --cov-fail-under=100 ofrak_type_test - fun-coverage --cov-fail-under=100 || ! (uname -m | grep x86_64) + fun-coverage --cov-fail-under=100 From ca44d3352274950001735d9edff2270c5191d9e1 Mon Sep 17 00:00:00 2001 From: Wyatt <53830972+whyitfor@users.noreply.github.com> Date: Tue, 15 Apr 2025 17:02:44 -0400 Subject: [PATCH 21/28] Support more arm64 toolchains --- ofrak_patch_maker/Dockerstub | 45 +++++++++++++------ .../test_m68k_toolchain.py | 2 +- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index 482ff66df..d2c3a0c6f 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -2,35 +2,53 @@ ARG TARGETARCH COPY ${PACKAGE_PATH}/conf/toolchain_docker_${TARGETARCH}.conf /etc/ofrak/toolchain.conf +RUN mkdir -p /opt/rbs/toolchain + # LLVM -RUN mkdir -p /opt/rbs/toolchain && \ +RUN if [ "$TARGETARCH" = "amd64" ]; then \ cd /tmp && \ wget https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -C /opt/rbs/toolchain && \ rm -rf clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ - mv /opt/rbs/toolchain/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu- /opt/rbs/toolchain/llvm_12.0.1 + mv /opt/rbs/toolchain/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu- /opt/rbs/toolchain/llvm_12.0.1; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + apt-get update && apt-get install -y libtinfo5 && \ + cd /tmp/ && \ + wget https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/clang+llvm-12.0.1-aarch64-linux-gnu.tar.xz --show-progress --progress=bar:force:noscroll && \ + tar xf clang+llvm-12.0.1-aarch64-linux-gnu.tar.xz -C /opt/rbs/toolchain && \ + rm -rf clang+llvm-12.0.1-aarch64-linux-gnu.tar.xz && \ + mv /opt/rbs/toolchain/clang+llvm-12.0.1-aarch64-linux-gnu /opt/rbs/toolchain/llvm_12.0.1; \ +fi; # ARM GNU NONE EABI -RUN cd /tmp && \ +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + cd /tmp && \ wget https://developer.arm.com/-/media/Files/downloads/gnu-rm/10-2020q4/gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2 --show-progress --progress=bar:force:noscroll && \ tar xf gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2 -C /opt/rbs/toolchain && \ - rm -rf gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2 + rm -rf gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget https://developer.arm.com/-/media/Files/downloads/gnu-rm/10-2020q4/gcc-arm-none-eabi-10-2020-q4-major-aarch64-linux.tar.bz2 --show-progress --progress=bar:force:noscroll && \ + tar xf gcc-arm-none-eabi-10-2020-q4-major-aarch64-linux.tar.bz2 -C /opt/rbs/toolchain && \ + rm -rf gcc-arm-none-eabi-10-2020-q4-major-aarch64-linux.tar.bz2; \ +fi; # LINUX GNU + BINUTILS -RUN cd /tmp && \ +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + cd /tmp && \ wget https://download.01.org/0day-ci/cross-package/gcc-10.3.0-nolibc/x86_64-gcc-10.3.0-nolibc_x86_64-linux.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf x86_64-gcc-10.3.0-nolibc_x86_64-linux.tar.xz -C /opt/rbs/toolchain && \ - rm -rf x86_64-gcc-10.3.0-nolibc_x86_64-linux.tar.xz - -#X64-64 toolchain for arm64 Docker images -RUN if [ "$TARGETARCH" = "arm64" ]; then \ - apt-get update && apt-get install -y gcc-12-x86-64-linux-gnu; \ + rm -rf x86_64-gcc-10.3.0-nolibc_x86_64-linux.tar.xz; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget https://www.kernel.org/pub/tools/crosstool/files/bin/arm64/10.3.0/arm64-gcc-10.3.0-nolibc-x86_64-linux.tar.xz --show-progress --progress=bar:force:noscroll && \ + tar xvf arm64-gcc-10.3.0-nolibc-x86_64-linux.tar.xz -C /opt/rbs/toolchain && \ + rm -rf arm64-gcc-10.3.0-nolibc-x86_64-linux.tar.xz; \ fi; #M68k GNU 10 Linux #Only exists for x86 -RUN if [ "$TARGETARCH" = "amd64" ]; then \ - cd /tmp && \ +RUN cd /tmp && \ apt-get update && apt-get install -y gcc g++ gperf bison flex texinfo help2man make libncurses5-dev python3-dev autoconf automake libtool libtool-bin gawk wget bzip2 xz-utils unzip patch libstdc++6 rsync && \ git clone https://github.com/crosstool-ng/crosstool-ng.git && \ cd crosstool-ng/ && \ @@ -84,8 +102,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ >> .config && \ ./ct-ng build CT_JOBS=`nproc` \ | sed 's/.*\r//g' && \ - cd /tmp && rm -rf crosstool-ng; \ -fi; + cd /tmp && rm -rf crosstool-ng #M68k VBCC RUN cd /tmp && \ diff --git a/ofrak_patch_maker/ofrak_patch_maker_test/test_m68k_toolchain.py b/ofrak_patch_maker/ofrak_patch_maker_test/test_m68k_toolchain.py index 739d7568a..56aa15db5 100644 --- a/ofrak_patch_maker/ofrak_patch_maker_test/test_m68k_toolchain.py +++ b/ofrak_patch_maker/ofrak_patch_maker_test/test_m68k_toolchain.py @@ -3,7 +3,6 @@ import pytest -from ofrak_type import ArchInfo from ofrak_patch_maker.model import PatchRegionConfig from ofrak_patch_maker.patch_maker import PatchMaker from ofrak_patch_maker.toolchain.gnu_m68k import GNU_M68K_LINUX_10_Toolchain @@ -21,6 +20,7 @@ run_monkey_patch_test, ) from ofrak_patch_maker_test.toolchain_c import run_hello_world_test, run_bounds_check_test +from ofrak_type import ArchInfo from ofrak_type.architecture import ( InstructionSet, ProcessorType, From 19ebf4bfc156e355fc6ecd6ab909237c5dd47282 Mon Sep 17 00:00:00 2001 From: Wyatt <53830972+whyitfor@users.noreply.github.com> Date: Tue, 15 Apr 2025 18:16:16 -0400 Subject: [PATCH 22/28] Use one toolchain.conf, standard toolchain directory names --- ofrak_patch_maker/Dockerstub | 7 +- .../conf/toolchain_docker_amd64.conf | 79 ------------------- .../conf/toolchain_docker_arm64.conf | 79 ------------------- .../ofrak_patch_maker/toolchain.conf | 33 +++----- 4 files changed, 16 insertions(+), 182 deletions(-) delete mode 100644 ofrak_patch_maker/conf/toolchain_docker_amd64.conf delete mode 100644 ofrak_patch_maker/conf/toolchain_docker_arm64.conf diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index d2c3a0c6f..d917bd66b 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -1,7 +1,5 @@ ARG TARGETARCH -COPY ${PACKAGE_PATH}/conf/toolchain_docker_${TARGETARCH}.conf /etc/ofrak/toolchain.conf - RUN mkdir -p /opt/rbs/toolchain # LLVM @@ -10,7 +8,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ wget https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -C /opt/rbs/toolchain && \ rm -rf clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ - mv /opt/rbs/toolchain/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu- /opt/rbs/toolchain/llvm_12.0.1; \ + mv /opt/rbs/toolchain/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu /opt/rbs/toolchain/llvm_12.0.1; \ elif [ "$TARGETARCH" = "arm64" ]; then \ apt-get update && apt-get install -y libtinfo5 && \ cd /tmp/ && \ @@ -127,11 +125,13 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ cd /tmp && \ wget https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz -C /opt/rbs/toolchain && \ + mv /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64 &&\ rm -rf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz; \ elif [ "$TARGETARCH" = "arm64" ]; then \ cd /tmp && \ wget https://developer.arm.com/-/media/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz -C /opt/rbs/toolchain && \ + mv /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64 &&\ rm -rf gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz; \ fi; @@ -140,6 +140,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ cd /tmp && \ wget https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/SoftwareLibraries/Firmware/avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz --show-progress --progress=bar:force:noscroll && \ tar xzf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz -C /opt/rbs/toolchain && \ + mv /opt/rbs/toolchain/avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64 /opt/rbs/toolchain/avr && \ rm -rf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz; \ elif [ "$TARGETARCH" = "arm64" ]; then \ cd /tmp && \ diff --git a/ofrak_patch_maker/conf/toolchain_docker_amd64.conf b/ofrak_patch_maker/conf/toolchain_docker_amd64.conf deleted file mode 100644 index 769dab9b9..000000000 --- a/ofrak_patch_maker/conf/toolchain_docker_amd64.conf +++ /dev/null @@ -1,79 +0,0 @@ -[LLVM_12_0_1] -PREPROCESSOR = /opt/rbs/toolchain/llvm_12.0.1/bin/clang -COMPILER = /opt/rbs/toolchain/llvm_12.0.1/bin/clang -LINKER = /opt/rbs/toolchain/llvm_12.0.1/bin/ld.lld -BIN_PARSER = /opt/rbs/toolchain/llvm_12.0.1/bin/llvm-readobj -LIB = /opt/rbs/toolchain/llvm_12.0.1/lib - -[GNU_ARM_NONE_EABI_10_2_1] -PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc -COMPILER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc -LINKER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-objdump -LIB = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/lib - -[GNU_M68K_LINUX_10] -PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc -COMPILER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc -LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld -BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump -LIB=/opt/rbs/toolchain/m68k-unknown-linux-gnu/lib - -[VBCC_M68K_0_9] -PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc -COMPILER = /opt/rbs/toolchain/vbcc_0_9/bin/vbccm68k -LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld -BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump - -[GNU_X86_64_LINUX_EABI_10_3_0] -# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler -PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc -COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc -LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump -LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib - -[GNU_X86_32_LINUX_EABI_10_3_0] -# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler -PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc -COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc -LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump -LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib - -[GNU_AARCH64_LINUX_10] -PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc -COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc -LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-objdump -LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/lib - -[GNU_AVR_5] -PREPROCESSOR = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc -COMPILER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc -LINKER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-ld -BIN_PARSER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-objdump -LIB = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/lib - -[GNU_PPC_LINUX_10] -PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc -COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc -LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-objdump -LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/lib - -[BCC_SPARC_GAISLER_ELF] -PREPROCESSOR = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc -COMPILER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc -LINKER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-ld -BIN_PARSER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-objdump -LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib - -[ASM] -ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as -X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as -M68K_ASM_PATH = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-as -AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-as -AVR_ASM_PATH = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-as -PPC_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-as -SPARC_ASM_PATH = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-as diff --git a/ofrak_patch_maker/conf/toolchain_docker_arm64.conf b/ofrak_patch_maker/conf/toolchain_docker_arm64.conf deleted file mode 100644 index 935f46f2b..000000000 --- a/ofrak_patch_maker/conf/toolchain_docker_arm64.conf +++ /dev/null @@ -1,79 +0,0 @@ -[LLVM_12_0_1] -PREPROCESSOR = /opt/rbs/toolchain/llvm_12.0.1/bin/clang -COMPILER = /opt/rbs/toolchain/llvm_12.0.1/bin/clang -LINKER = /opt/rbs/toolchain/llvm_12.0.1/bin/ld.lld -BIN_PARSER = /opt/rbs/toolchain/llvm_12.0.1/bin/llvm-readobj -LIB = /opt/rbs/toolchain/llvm_12.0.1/lib - -[GNU_ARM_NONE_EABI_10_2_1] -PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc -COMPILER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc -LINKER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-objdump -LIB = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/lib - -[GNU_M68K_LINUX_10] -PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc -COMPILER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc -LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld -BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump -LIB=/opt/rbs/toolchain/m68k-unknown-linux-gnu/lib - -[VBCC_M68K_0_9] -PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc -COMPILER = /opt/rbs/toolchain/vbcc_0_9/bin/vbccm68k -LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld -BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump - -[GNU_X86_64_LINUX_EABI_10_3_0] -# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler -PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc -COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc -LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump -LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib - -[GNU_X86_32_LINUX_EABI_10_3_0] -# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler -PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc -COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc -LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump -LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib - -[GNU_AARCH64_LINUX_10] -PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-gcc -COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-gcc -LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-objdump -LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/lib - -[GNU_AVR_5] -PREPROCESSOR = /opt/rbs/toolchain/avr/bin/avr-gcc -COMPILER = /opt/rbs/toolchain/avr/bin/avr-gcc -LINKER = /opt/rbs/toolchain/avr/bin/avr-ld -BIN_PARSER = /opt/rbs/toolchain/avr/bin/avr-objdump -LIB = /opt/rbs/toolchain/avr/lib - -[GNU_PPC_LINUX_10] -PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc -COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc -LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-objdump -LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/lib - -[BCC_SPARC_GAISLER_ELF] -PREPROCESSOR = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc -COMPILER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc -LINKER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-ld -BIN_PARSER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-objdump -LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib - -[ASM] -ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as -X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as -M68K_ASM_PATH = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-as -AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-as -AVR_ASM_PATH = /opt/rbs/toolchain/avr/bin/avr-as -PPC_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-as -SPARC_ASM_PATH = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-as diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf index 769dab9b9..c2fe2102e 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf @@ -26,7 +26,6 @@ LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump [GNU_X86_64_LINUX_EABI_10_3_0] -# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld @@ -34,7 +33,6 @@ BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux- LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib [GNU_X86_32_LINUX_EABI_10_3_0] -# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld @@ -42,18 +40,18 @@ BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux- LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib [GNU_AARCH64_LINUX_10] -PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc -COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc -LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-objdump -LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/lib +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/bin/aarch64-none-elf-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/bin/aarch64-none-elf-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/bin/aarch64-none-elf-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/bin/aarch64-none-elf-objdump +LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/lib [GNU_AVR_5] -PREPROCESSOR = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc -COMPILER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc -LINKER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-ld -BIN_PARSER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-objdump -LIB = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/lib +PREPROCESSOR = /opt/rbs/toolchain/avr/bin/avr-gcc +COMPILER = /opt/rbs/toolchain/avr/bin/avr-gcc +LINKER = /opt/rbs/toolchain/avr/bin/avr-ld +BIN_PARSER = /opt/rbs/toolchain/avr/bin/avr-objdump +LIB = /opt/rbs/toolchain/avr/lib [GNU_PPC_LINUX_10] PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc @@ -62,18 +60,11 @@ LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-ld BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-objdump LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/lib -[BCC_SPARC_GAISLER_ELF] -PREPROCESSOR = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc -COMPILER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc -LINKER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-ld -BIN_PARSER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-objdump -LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib - [ASM] ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as M68K_ASM_PATH = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-as -AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-as -AVR_ASM_PATH = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-as +AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/bin/aarch64-none-elf-as +AVR_ASM_PATH = /opt/rbs/toolchain/avr/bin/avr-as PPC_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-as SPARC_ASM_PATH = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-as From 3bb7b1aafafc9bf396a054d1d85919beef740cef Mon Sep 17 00:00:00 2001 From: Wyatt <53830972+whyitfor@users.noreply.github.com> Date: Tue, 15 Apr 2025 18:33:08 -0400 Subject: [PATCH 23/28] Revert erroneous change --- ofrak_patch_maker/Dockerstub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index d917bd66b..1bc284923 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -8,7 +8,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ wget https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -C /opt/rbs/toolchain && \ rm -rf clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ - mv /opt/rbs/toolchain/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu /opt/rbs/toolchain/llvm_12.0.1; \ + mv /opt/rbs/toolchain/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu- /opt/rbs/toolchain/llvm_12.0.1; \ elif [ "$TARGETARCH" = "arm64" ]; then \ apt-get update && apt-get install -y libtinfo5 && \ cd /tmp/ && \ From 82a705b0d9a5229453a034d2bc2e7be1a3c55a0f Mon Sep 17 00:00:00 2001 From: Wyatt <53830972+whyitfor@users.noreply.github.com> Date: Tue, 15 Apr 2025 18:53:47 -0400 Subject: [PATCH 24/28] Update changelogs, adjust typo --- disassemblers/ofrak_capstone/CHANGELOG.md | 4 ++-- disassemblers/ofrak_capstone/setup.py | 2 +- ofrak_core/CHANGELOG.md | 5 +++-- ofrak_core/setup.py | 4 ++-- ofrak_patch_maker/CHANGELOG.md | 6 +++--- ofrak_patch_maker/Dockerstub | 10 +++++----- ofrak_patch_maker/ofrak_patch_maker/toolchain.conf | 9 ++++++++- ofrak_patch_maker/setup.py | 2 +- 8 files changed, 25 insertions(+), 17 deletions(-) diff --git a/disassemblers/ofrak_capstone/CHANGELOG.md b/disassemblers/ofrak_capstone/CHANGELOG.md index a3f1ed008..49626bd4e 100644 --- a/disassemblers/ofrak_capstone/CHANGELOG.md +++ b/disassemblers/ofrak_capstone/CHANGELOG.md @@ -3,13 +3,13 @@ All notable changes to `ofrak-capstone` will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased 1.1.0rc0](https://github.com/redballoonsecurity/ofrak/tree/master) +## [Unreleased 1.1.0rc1](https://github.com/redballoonsecurity/ofrak/tree/master) ### Fixed - Capstone tests fixed to use `run_instruction_unpacker_test_case` ([#503](https://github.com/redballoonsecurity/ofrak/pull/503)) ### Changed -- Update to captione==5.0.0.post1. +- Update to captione==5.0.3. - Minor update to OFRAK Community License, add OFRAK Pro License ([#478](https://github.com/redballoonsecurity/ofrak/pull/478)) - Updated library to support ofrak 3.3.0rc0. ([#539](https://github.com/redballoonsecurity/ofrak/issues/539)) diff --git a/disassemblers/ofrak_capstone/setup.py b/disassemblers/ofrak_capstone/setup.py index f49dc62bc..1f895122f 100644 --- a/disassemblers/ofrak_capstone/setup.py +++ b/disassemblers/ofrak_capstone/setup.py @@ -31,7 +31,7 @@ def read_requirements(requirements_path): setuptools.setup( name="ofrak_capstone", - version="1.0.0", + version="1.1.0rc1", packages=setuptools.find_packages(exclude=["ofrak_capstone_test", "ofrak_capstone_test.*"]), package_data={"ofrak_capstone": ["py.typed"]}, install_requires=["ofrak>=3.3.0rc0"] + read_requirements("requirements.txt"), diff --git a/ofrak_core/CHANGELOG.md b/ofrak_core/CHANGELOG.md index 980b18cb1..2cff1980f 100644 --- a/ofrak_core/CHANGELOG.md +++ b/ofrak_core/CHANGELOG.md @@ -3,7 +3,7 @@ All notable changes to `ofrak` will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased: 3.3.0rc2](https://github.com/redballoonsecurity/ofrak/tree/master) +## [Unreleased: 3.3.0rc3](https://github.com/redballoonsecurity/ofrak/tree/master) ### Added - Add license check command to prompt users about community or pro licenses. ([#478](https://github.com/redballoonsecurity/ofrak/pull/478)) - Support `application/vnd.android.package-archive` mime type for APKs, which is returned by newer versions of libmagic ([#470](https://github.com/redballoonsecurity/ofrak/pull/470)) @@ -67,6 +67,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Add `RawMagicPattern` to efficiently run custom magic byte search logic within `MagicIdenfifier` - Update registered identifiers to make use of new `MagicIdentifier` for following resource tags: `Apk`, `Bzip2Data`, `CpioFilesystem`, `DeviceTreeBlob`, `Elf`, `Ext2Filesystem`, `Ext3Filesystem`, `Ext4Filesystem`, `GzipData`, `ISO9660Image`, `Jffs2Filesystem`, `LzmaData`, `XzData`, `LzoData`, `OpenWrtTrx`, `Pe`, `RarArchive`, `SevenZFilesystem`, `SquashfsFilesystem`, `TarArchive`, `Ubi`, `Ubifs`, `Uf2File`, `UImage`, `ZipArchive`, `ZlibData`, `ZstdData` - Update `Instruction.get_assembly` to by synchronous ([#539](https://github.com/redballoonsecurity/ofrak/issues/539)) +- Update orjson to ~=3.10.12 ([#562](https://github.com/redballoonsecurity/ofrak/pull/562/files)) ### Deprecated - `Resource.flush_to_disk` deprecated in favor of `Resource.flush_data_to_disk`. ([#373](https://github.com/redballoonsecurity/ofrak/pull/373), [#567](https://github.com/redballoonsecurity/ofrak/pull/568)) @@ -77,7 +78,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Security - Update aiohttp to 3.10.11 ([#522](https://github.com/redballoonsecurity/ofrak/pull/522)) - Update pycryptogrpahy to version 43.0.3. ([#525](https://github.com/redballoonsecurity/ofrak/pull/525)) -- Bump `lief` dependency to 0.15.1 to address [vulnerability](https://github.com/redballoonsecurity/ofrak/security/dependabot/31) in lower versions ([#502](https://github.com/redballoonsecurity/ofrak/pull/502)) +- Bump `lief` dependency to 0.16.1 to address [vulnerability](https://github.com/redballoonsecurity/ofrak/security/dependabot/31) in lower versions ([#502](https://github.com/redballoonsecurity/ofrak/pull/502), [#562](https://github.com/redballoonsecurity/ofrak/pull/562/files)) ## [3.2.0](https://github.com/redballoonsecurity/ofrak/compare/ofrak-v3.1.0...ofrak-v3.2.0) ### Added diff --git a/ofrak_core/setup.py b/ofrak_core/setup.py index 8504489d0..ef27c580c 100644 --- a/ofrak_core/setup.py +++ b/ofrak_core/setup.py @@ -69,7 +69,7 @@ def read_requirements(requirements_path): setuptools.setup( name="ofrak", - version="3.3.0rc2", + version="3.3.0rc3", description="A binary analysis and modification platform", packages=setuptools.find_packages(exclude=["test_ofrak", "test_ofrak.*"]), package_data={ @@ -78,7 +78,7 @@ def read_requirements(requirements_path): install_requires=[ "ofrak_io>=1.0,==1.*", "ofrak_type>=2.2.0rc0,==2.*", - "ofrak_patch_maker>=4.0.2rc0,==4.*", + "ofrak_patch_maker>=4.1.0rc0,==4.*", ] + read_requirements("requirements.txt"), extras_require={ diff --git a/ofrak_patch_maker/CHANGELOG.md b/ofrak_patch_maker/CHANGELOG.md index f8ebe2230..7a7bb2a41 100644 --- a/ofrak_patch_maker/CHANGELOG.md +++ b/ofrak_patch_maker/CHANGELOG.md @@ -3,10 +3,10 @@ All notable changes to `ofrak-patch-maker` will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) +## [Unreleased: 4.1.0rc0](https://github.com/redballoonsecurity/ofrak/tree/master) ### Added - GNU 32-bit x86 toolchain. ([#405](https://github.com/redballoonsecurity/ofrak/pull/405)) -- SPARC BCC tolchain. ([#462](https://github.com/redballoonsecurity/ofrak/pull/462)) +- SPARC BCC toolchain (x86_64 platforms only) ([#462](https://github.com/redballoonsecurity/ofrak/pull/462), [#405](https://github.com/redballoonsecurity/ofrak/pull/405)) ### Removed - Switch from binutils version of linker, bin_parser, assembler for X86_64 to default Debian versions. ([#405](https://github.com/redballoonsecurity/ofrak/pull/405)) @@ -18,7 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Changed - Removed `SUBALIGN(0)` for `.bss` sections - Minor update to OFRAK Community License, add OFRAK Pro License ([#478](https://github.com/redballoonsecurity/ofrak/pull/478)) -- Install toolchains from source or tarball instead of relying on the Debian package manager ([#502](https://github.com/redballoonsecurity/ofrak/pull/502), [#541](https://github.com/redballoonsecurity/ofrak/pull/541)) +- Install toolchains (x86_64, aarch64 platforms) from source or tarball instead of relying on the Debian package manager ([#405](https://github.com/redballoonsecurity/ofrak/pull/405), [#502](https://github.com/redballoonsecurity/ofrak/pull/502), [#541](https://github.com/redballoonsecurity/ofrak/pull/541), [#562](https://github.com/redballoonsecurity/ofrak/pull/562/files)) - Deprecate reserving `.bss` space at the FEM linking step with `create_unsafe_bss_segment` method and `unsafe_bss_segment` argument. ([#505](https://github.com/redballoonsecurity/ofrak/pull/505)) - Determine the correct allocation behavior of sections from ELF flags rather than section names. ([#505](https://github.com/redballoonsecurity/ofrak/pull/505)) diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index 1bc284923..2d5fd3570 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -123,10 +123,10 @@ RUN cd /tmp && \ #AARCH64 GNU 10 Linux RUN if [ "$TARGETARCH" = "amd64" ]; then \ cd /tmp && \ - wget https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz --show-progress --progress=bar:force:noscroll && \ - tar xf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz -C /opt/rbs/toolchain && \ - mv /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64 &&\ - rm -rf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz; \ + wget https://developer.arm.com/-/media/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz --show-progress --progress=bar:force:noscroll && \ + tar xf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz -C /opt/rbs/toolchain && \ + mv /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64 &&\ + rm -rf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz; \ elif [ "$TARGETARCH" = "arm64" ]; then \ cd /tmp && \ wget https://developer.arm.com/-/media/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz --show-progress --progress=bar:force:noscroll && \ @@ -140,7 +140,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ cd /tmp && \ wget https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/SoftwareLibraries/Firmware/avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz --show-progress --progress=bar:force:noscroll && \ tar xzf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz -C /opt/rbs/toolchain && \ - mv /opt/rbs/toolchain/avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64 /opt/rbs/toolchain/avr && \ + mv /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64 /opt/rbs/toolchain/avr && \ rm -rf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz; \ elif [ "$TARGETARCH" = "arm64" ]; then \ cd /tmp && \ diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf index c2fe2102e..afd212d7a 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf @@ -44,7 +44,7 @@ PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/bin/aarch64-none- COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/bin/aarch64-none-elf-gcc LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/bin/aarch64-none-elf-ld BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/bin/aarch64-none-elf-objdump -LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/lib +LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/lib [GNU_AVR_5] PREPROCESSOR = /opt/rbs/toolchain/avr/bin/avr-gcc @@ -60,6 +60,13 @@ LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-ld BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-objdump LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/lib +[BCC_SPARC_GAISLER_ELF] +PREPROCESSOR = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc +COMPILER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc +LINKER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-ld +BIN_PARSER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-objdump +LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib + [ASM] ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as diff --git a/ofrak_patch_maker/setup.py b/ofrak_patch_maker/setup.py index d135259fb..2b81ab4fd 100644 --- a/ofrak_patch_maker/setup.py +++ b/ofrak_patch_maker/setup.py @@ -31,7 +31,7 @@ def read_requirements(requirements_path): setuptools.setup( name="ofrak_patch_maker", - version="4.0.2", + version="4.1.0rc0", description="PatchMaker tool for applying source-code patches to binaries", packages=setuptools.find_packages(exclude=("ofrak_patch_maker_test",)), package_data={"ofrak_patch_maker": ["py.typed"]}, From 31ae6547ac529201f0bd9501c9de566330f5960c Mon Sep 17 00:00:00 2001 From: Wyatt <53830972+whyitfor@users.noreply.github.com> Date: Thu, 17 Apr 2025 17:14:02 -0400 Subject: [PATCH 25/28] Break toolchain configs back out based on platform, add default toolchain.conf for Python installs --- ofrak_core/Dockerstub | 6 +- ofrak_patch_maker/Dockerstub | 11 ++- .../config/amd64_platform_toolchain.conf | 77 ++++++++++++++++++ .../arm64_platform_toolchain.conf} | 9 ++- .../ofrak_patch_maker/config/toolchain.conf | 81 +++++++++++++++++++ 5 files changed, 171 insertions(+), 13 deletions(-) create mode 100644 ofrak_patch_maker/ofrak_patch_maker/config/amd64_platform_toolchain.conf rename ofrak_patch_maker/ofrak_patch_maker/{toolchain.conf => config/arm64_platform_toolchain.conf} (90%) create mode 100644 ofrak_patch_maker/ofrak_patch_maker/config/toolchain.conf diff --git a/ofrak_core/Dockerstub b/ofrak_core/Dockerstub index d2d76bfac..16fd6cde4 100644 --- a/ofrak_core/Dockerstub +++ b/ofrak_core/Dockerstub @@ -24,9 +24,9 @@ RUN python3 -m pip install python-lzo # Install apktool and uber-apk-signer RUN apt-get -y update && apt-get -y install openjdk-17-jdk -RUN wget https://raw.githubusercontent.com/iBotPeaches/Apktool/v2.3.3/scripts/linux/apktool -O /usr/local/bin/apktool && \ - wget https://bitbucket.org/iBotPeaches/apktool/downloads/apktool_2.3.3.jar -O /usr/local/bin/apktool.jar && \ - wget https://github.com/patrickfav/uber-apk-signer/releases/download/v1.0.0/uber-apk-signer-1.0.0.jar -O /usr/local/bin/uber-apk-signer.jar && \ +RUN wget https://raw.githubusercontent.com/iBotPeaches/Apktool/v2.11.1/scripts/linux/apktool -O /usr/local/bin/apktool && \ + wget https://bitbucket.org/iBotPeaches/apktool/downloads/apktool_2.11.1.jar -O /usr/local/bin/apktool.jar && \ + wget https://github.com/patrickfav/uber-apk-signer/releases/download/v1.3.0/uber-apk-signer-1.3.0.jar -O /usr/local/bin/uber-apk-signer.jar && \ chmod +x /usr/local/bin/apktool* # Install official 7-zip diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index 2d5fd3570..cf6d991cf 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -2,6 +2,8 @@ ARG TARGETARCH RUN mkdir -p /opt/rbs/toolchain +COPY ${PACKAGE_PATH}/ofrak_patch_maker/config/${TARGETARCH}_platform_toolchain.conf /etc/ofrak/toolchain.conf + # LLVM RUN if [ "$TARGETARCH" = "amd64" ]; then \ cd /tmp && \ @@ -123,15 +125,13 @@ RUN cd /tmp && \ #AARCH64 GNU 10 Linux RUN if [ "$TARGETARCH" = "amd64" ]; then \ cd /tmp && \ - wget https://developer.arm.com/-/media/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz --show-progress --progress=bar:force:noscroll && \ - tar xf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz -C /opt/rbs/toolchain && \ - mv /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64 &&\ - rm -rf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz; \ + wget https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz --show-progress --progress=bar:force:noscroll && \ + tar xf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz -C /opt/rbs/toolchain && \ + rm -rf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz; \ elif [ "$TARGETARCH" = "arm64" ]; then \ cd /tmp && \ wget https://developer.arm.com/-/media/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz -C /opt/rbs/toolchain && \ - mv /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64 &&\ rm -rf gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz; \ fi; @@ -140,7 +140,6 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ cd /tmp && \ wget https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/SoftwareLibraries/Firmware/avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz --show-progress --progress=bar:force:noscroll && \ tar xzf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz -C /opt/rbs/toolchain && \ - mv /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64 /opt/rbs/toolchain/avr && \ rm -rf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz; \ elif [ "$TARGETARCH" = "arm64" ]; then \ cd /tmp && \ diff --git a/ofrak_patch_maker/ofrak_patch_maker/config/amd64_platform_toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/config/amd64_platform_toolchain.conf new file mode 100644 index 000000000..410335511 --- /dev/null +++ b/ofrak_patch_maker/ofrak_patch_maker/config/amd64_platform_toolchain.conf @@ -0,0 +1,77 @@ +[LLVM_12_0_1] +PREPROCESSOR = /opt/rbs/toolchain/llvm_12.0.1/bin/clang +COMPILER = /opt/rbs/toolchain/llvm_12.0.1/bin/clang +LINKER = /opt/rbs/toolchain/llvm_12.0.1/bin/ld.lld +BIN_PARSER = /opt/rbs/toolchain/llvm_12.0.1/bin/llvm-readobj +LIB = /opt/rbs/toolchain/llvm_12.0.1/lib + +[GNU_ARM_NONE_EABI_10_2_1] +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-objdump +LIB = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/lib + +[GNU_M68K_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump +LIB=/opt/rbs/toolchain/m68k-unknown-linux-gnu/lib + +[VBCC_M68K_0_9] +PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/vbcc_0_9/bin/vbccm68k +LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump + +[GNU_X86_64_LINUX_EABI_10_3_0] +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib + +[GNU_X86_32_LINUX_EABI_10_3_0] +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib + +[GNU_AARCH64_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-objdump +LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/lib + +[GNU_AVR_5] +PREPROCESSOR = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc +COMPILER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc +LINKER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-ld +BIN_PARSER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-objdump +LIB = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/lib + +[GNU_PPC_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/lib + +[BCC_SPARC_GAISLER_ELF] +PREPROCESSOR = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc +COMPILER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc +LINKER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-ld +BIN_PARSER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-objdump +LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib + +[ASM] +ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as +X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as +M68K_ASM_PATH = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-as +AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-as +AVR_ASM_PATH = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-as +PPC_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-as +SPARC_ASM_PATH = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-as diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/config/arm64_platform_toolchain.conf similarity index 90% rename from ofrak_patch_maker/ofrak_patch_maker/toolchain.conf rename to ofrak_patch_maker/ofrak_patch_maker/config/arm64_platform_toolchain.conf index afd212d7a..d594387bc 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf +++ b/ofrak_patch_maker/ofrak_patch_maker/config/arm64_platform_toolchain.conf @@ -1,3 +1,4 @@ +# Toolchain config for X86_64 Platforms. [LLVM_12_0_1] PREPROCESSOR = /opt/rbs/toolchain/llvm_12.0.1/bin/clang COMPILER = /opt/rbs/toolchain/llvm_12.0.1/bin/clang @@ -41,10 +42,10 @@ LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib [GNU_AARCH64_LINUX_10] PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/bin/aarch64-none-elf-gcc -COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/bin/aarch64-none-elf-gcc -LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/bin/aarch64-none-elf-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/bin/aarch64-none-elf-objdump -LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/lib +COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-objdump +LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/lib [GNU_AVR_5] PREPROCESSOR = /opt/rbs/toolchain/avr/bin/avr-gcc diff --git a/ofrak_patch_maker/ofrak_patch_maker/config/toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/config/toolchain.conf new file mode 100644 index 000000000..132684108 --- /dev/null +++ b/ofrak_patch_maker/ofrak_patch_maker/config/toolchain.conf @@ -0,0 +1,81 @@ +# This file is the default toolchain config, to specify what toolchains ofrak_patch_maker uses. +# See "amd64_platform_toolchain.conf" or "arm64_platform_toolchain.conf" for examples of full +# toolchain configs used in OFRAK Docker images. +[LLVM_12_0_1] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[GNU_ARM_NONE_EABI_10_2_1] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[GNU_M68K_LINUX_10] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[VBCC_M68K_0_9] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[GNU_X86_64_LINUX_EABI_10_3_0] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[GNU_X86_32_LINUX_EABI_10_3_0] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[GNU_AARCH64_LINUX_10] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[GNU_AVR_5] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[GNU_PPC_LINUX_10] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[BCC_SPARC_GAISLER_ELF] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[ASM] +ARM_ASM_PATH = +X86_64_ASM_PATH = +M68K_ASM_PATH = +AARCH64_ASM_PATH = +AVR_ASM_PATH = +PPC_ASM_PATH = +SPARC_ASM_PATH = From d9bd3093bdf7c4ae6149b7adc89c202c5777afc2 Mon Sep 17 00:00:00 2001 From: Wyatt <53830972+whyitfor@users.noreply.github.com> Date: Fri, 18 Apr 2025 07:00:35 -0400 Subject: [PATCH 26/28] Fix typo, inadvertent change to ofrak_core/Dockerstub --- ofrak_core/Dockerstub | 6 +++--- .../ofrak_patch_maker/config/arm64_platform_toolchain.conf | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ofrak_core/Dockerstub b/ofrak_core/Dockerstub index 16fd6cde4..d2d76bfac 100644 --- a/ofrak_core/Dockerstub +++ b/ofrak_core/Dockerstub @@ -24,9 +24,9 @@ RUN python3 -m pip install python-lzo # Install apktool and uber-apk-signer RUN apt-get -y update && apt-get -y install openjdk-17-jdk -RUN wget https://raw.githubusercontent.com/iBotPeaches/Apktool/v2.11.1/scripts/linux/apktool -O /usr/local/bin/apktool && \ - wget https://bitbucket.org/iBotPeaches/apktool/downloads/apktool_2.11.1.jar -O /usr/local/bin/apktool.jar && \ - wget https://github.com/patrickfav/uber-apk-signer/releases/download/v1.3.0/uber-apk-signer-1.3.0.jar -O /usr/local/bin/uber-apk-signer.jar && \ +RUN wget https://raw.githubusercontent.com/iBotPeaches/Apktool/v2.3.3/scripts/linux/apktool -O /usr/local/bin/apktool && \ + wget https://bitbucket.org/iBotPeaches/apktool/downloads/apktool_2.3.3.jar -O /usr/local/bin/apktool.jar && \ + wget https://github.com/patrickfav/uber-apk-signer/releases/download/v1.0.0/uber-apk-signer-1.0.0.jar -O /usr/local/bin/uber-apk-signer.jar && \ chmod +x /usr/local/bin/apktool* # Install official 7-zip diff --git a/ofrak_patch_maker/ofrak_patch_maker/config/arm64_platform_toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/config/arm64_platform_toolchain.conf index d594387bc..1c3f4ad6d 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/config/arm64_platform_toolchain.conf +++ b/ofrak_patch_maker/ofrak_patch_maker/config/arm64_platform_toolchain.conf @@ -41,7 +41,7 @@ BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux- LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib [GNU_AARCH64_LINUX_10] -PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/bin/aarch64-none-elf-gcc +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-gcc COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-gcc LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-ld BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-objdump @@ -72,7 +72,7 @@ LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as M68K_ASM_PATH = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-as -AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64/bin/aarch64-none-elf-as +AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-as AVR_ASM_PATH = /opt/rbs/toolchain/avr/bin/avr-as PPC_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-as SPARC_ASM_PATH = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-as From 2893f6f72df0acedb642b74dffee7f65b311fc2f Mon Sep 17 00:00:00 2001 From: Wyatt <53830972+whyitfor@users.noreply.github.com> Date: Sat, 19 Apr 2025 06:55:45 -0400 Subject: [PATCH 27/28] Edit comment --- ofrak_patch_maker/Dockerstub | 1 - 1 file changed, 1 deletion(-) diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index cf6d991cf..727941fad 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -47,7 +47,6 @@ elif [ "$TARGETARCH" = "arm64" ]; then \ fi; #M68k GNU 10 Linux -#Only exists for x86 RUN cd /tmp && \ apt-get update && apt-get install -y gcc g++ gperf bison flex texinfo help2man make libncurses5-dev python3-dev autoconf automake libtool libtool-bin gawk wget bzip2 xz-utils unzip patch libstdc++6 rsync && \ git clone https://github.com/crosstool-ng/crosstool-ng.git && \ From 79da6a4d02988ecbff6fdabdebd57502cfabccfd Mon Sep 17 00:00:00 2001 From: Wyatt <53830972+whyitfor@users.noreply.github.com> Date: Mon, 21 Apr 2025 10:43:11 -0400 Subject: [PATCH 28/28] Make toolchain confs explicitly used in Docker; fix typos --- ofrak_patch_maker/Dockerstub | 2 +- .../config/{ => docker}/amd64_platform_toolchain.conf | 0 .../config/{ => docker}/arm64_platform_toolchain.conf | 0 ofrak_patch_maker/ofrak_patch_maker/config/toolchain.conf | 4 ++-- 4 files changed, 3 insertions(+), 3 deletions(-) rename ofrak_patch_maker/ofrak_patch_maker/config/{ => docker}/amd64_platform_toolchain.conf (100%) rename ofrak_patch_maker/ofrak_patch_maker/config/{ => docker}/arm64_platform_toolchain.conf (100%) diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index 727941fad..c05d3d065 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -2,7 +2,7 @@ ARG TARGETARCH RUN mkdir -p /opt/rbs/toolchain -COPY ${PACKAGE_PATH}/ofrak_patch_maker/config/${TARGETARCH}_platform_toolchain.conf /etc/ofrak/toolchain.conf +COPY ${PACKAGE_PATH}/ofrak_patch_maker/config/docker/${TARGETARCH}_platform_toolchain.conf /etc/ofrak/toolchain.conf # LLVM RUN if [ "$TARGETARCH" = "amd64" ]; then \ diff --git a/ofrak_patch_maker/ofrak_patch_maker/config/amd64_platform_toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/config/docker/amd64_platform_toolchain.conf similarity index 100% rename from ofrak_patch_maker/ofrak_patch_maker/config/amd64_platform_toolchain.conf rename to ofrak_patch_maker/ofrak_patch_maker/config/docker/amd64_platform_toolchain.conf diff --git a/ofrak_patch_maker/ofrak_patch_maker/config/arm64_platform_toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/config/docker/arm64_platform_toolchain.conf similarity index 100% rename from ofrak_patch_maker/ofrak_patch_maker/config/arm64_platform_toolchain.conf rename to ofrak_patch_maker/ofrak_patch_maker/config/docker/arm64_platform_toolchain.conf diff --git a/ofrak_patch_maker/ofrak_patch_maker/config/toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/config/toolchain.conf index 132684108..a2fd54135 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/config/toolchain.conf +++ b/ofrak_patch_maker/ofrak_patch_maker/config/toolchain.conf @@ -1,6 +1,6 @@ # This file is the default toolchain config, to specify what toolchains ofrak_patch_maker uses. -# See "amd64_platform_toolchain.conf" or "arm64_platform_toolchain.conf" for examples of full -# toolchain configs used in OFRAK Docker images. +# See "docker/amd64_platform_toolchain.conf" or "docker/arm64_platform_toolchain.conf" for examples +# of full toolchain configs used in OFRAK Docker images. [LLVM_12_0_1] PREPROCESSOR = COMPILER =