diff --git a/disassemblers/ofrak_angr/CHANGELOG.md b/disassemblers/ofrak_angr/CHANGELOG.md index 061a5550b..16259123b 100644 --- a/disassemblers/ofrak_angr/CHANGELOG.md +++ b/disassemblers/ofrak_angr/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) ### Changed -- Update to latest angr==9.2.77, which also necessitates Python >= 3.8. +- Update to latest angr==9.2.93, which also necessitates Python >= 3.8. - Refactored AngrDecompilationAnalysis/Analyzer to use generic components in ofrak core. ([#453](https://github.com/redballoonsecurity/ofrak/pull/453)) - Minor update to OFRAK Community License, add OFRAK Pro License ([#478](https://github.com/redballoonsecurity/ofrak/pull/478)) diff --git a/disassemblers/ofrak_capstone/CHANGELOG.md b/disassemblers/ofrak_capstone/CHANGELOG.md index a3f1ed008..49626bd4e 100644 --- a/disassemblers/ofrak_capstone/CHANGELOG.md +++ b/disassemblers/ofrak_capstone/CHANGELOG.md @@ -3,13 +3,13 @@ All notable changes to `ofrak-capstone` will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased 1.1.0rc0](https://github.com/redballoonsecurity/ofrak/tree/master) +## [Unreleased 1.1.0rc1](https://github.com/redballoonsecurity/ofrak/tree/master) ### Fixed - Capstone tests fixed to use `run_instruction_unpacker_test_case` ([#503](https://github.com/redballoonsecurity/ofrak/pull/503)) ### Changed -- Update to captione==5.0.0.post1. +- Update to captione==5.0.3. - Minor update to OFRAK Community License, add OFRAK Pro License ([#478](https://github.com/redballoonsecurity/ofrak/pull/478)) - Updated library to support ofrak 3.3.0rc0. ([#539](https://github.com/redballoonsecurity/ofrak/issues/539)) diff --git a/disassemblers/ofrak_capstone/requirements.txt b/disassemblers/ofrak_capstone/requirements.txt index 5903f94cb..22763a26e 100644 --- a/disassemblers/ofrak_capstone/requirements.txt +++ b/disassemblers/ofrak_capstone/requirements.txt @@ -1 +1 @@ -capstone==5.0.0.post1 +capstone==5.0.3 diff --git a/disassemblers/ofrak_capstone/setup.py b/disassemblers/ofrak_capstone/setup.py index f49dc62bc..1f895122f 100644 --- a/disassemblers/ofrak_capstone/setup.py +++ b/disassemblers/ofrak_capstone/setup.py @@ -31,7 +31,7 @@ def read_requirements(requirements_path): setuptools.setup( name="ofrak_capstone", - version="1.0.0", + version="1.1.0rc1", packages=setuptools.find_packages(exclude=["ofrak_capstone_test", "ofrak_capstone_test.*"]), package_data={"ofrak_capstone": ["py.typed"]}, install_requires=["ofrak>=3.3.0rc0"] + read_requirements("requirements.txt"), diff --git a/ofrak_core/CHANGELOG.md b/ofrak_core/CHANGELOG.md index 980b18cb1..2cff1980f 100644 --- a/ofrak_core/CHANGELOG.md +++ b/ofrak_core/CHANGELOG.md @@ -3,7 +3,7 @@ All notable changes to `ofrak` will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased: 3.3.0rc2](https://github.com/redballoonsecurity/ofrak/tree/master) +## [Unreleased: 3.3.0rc3](https://github.com/redballoonsecurity/ofrak/tree/master) ### Added - Add license check command to prompt users about community or pro licenses. ([#478](https://github.com/redballoonsecurity/ofrak/pull/478)) - Support `application/vnd.android.package-archive` mime type for APKs, which is returned by newer versions of libmagic ([#470](https://github.com/redballoonsecurity/ofrak/pull/470)) @@ -67,6 +67,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Add `RawMagicPattern` to efficiently run custom magic byte search logic within `MagicIdenfifier` - Update registered identifiers to make use of new `MagicIdentifier` for following resource tags: `Apk`, `Bzip2Data`, `CpioFilesystem`, `DeviceTreeBlob`, `Elf`, `Ext2Filesystem`, `Ext3Filesystem`, `Ext4Filesystem`, `GzipData`, `ISO9660Image`, `Jffs2Filesystem`, `LzmaData`, `XzData`, `LzoData`, `OpenWrtTrx`, `Pe`, `RarArchive`, `SevenZFilesystem`, `SquashfsFilesystem`, `TarArchive`, `Ubi`, `Ubifs`, `Uf2File`, `UImage`, `ZipArchive`, `ZlibData`, `ZstdData` - Update `Instruction.get_assembly` to by synchronous ([#539](https://github.com/redballoonsecurity/ofrak/issues/539)) +- Update orjson to ~=3.10.12 ([#562](https://github.com/redballoonsecurity/ofrak/pull/562/files)) ### Deprecated - `Resource.flush_to_disk` deprecated in favor of `Resource.flush_data_to_disk`. ([#373](https://github.com/redballoonsecurity/ofrak/pull/373), [#567](https://github.com/redballoonsecurity/ofrak/pull/568)) @@ -77,7 +78,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Security - Update aiohttp to 3.10.11 ([#522](https://github.com/redballoonsecurity/ofrak/pull/522)) - Update pycryptogrpahy to version 43.0.3. ([#525](https://github.com/redballoonsecurity/ofrak/pull/525)) -- Bump `lief` dependency to 0.15.1 to address [vulnerability](https://github.com/redballoonsecurity/ofrak/security/dependabot/31) in lower versions ([#502](https://github.com/redballoonsecurity/ofrak/pull/502)) +- Bump `lief` dependency to 0.16.1 to address [vulnerability](https://github.com/redballoonsecurity/ofrak/security/dependabot/31) in lower versions ([#502](https://github.com/redballoonsecurity/ofrak/pull/502), [#562](https://github.com/redballoonsecurity/ofrak/pull/562/files)) ## [3.2.0](https://github.com/redballoonsecurity/ofrak/compare/ofrak-v3.1.0...ofrak-v3.2.0) ### Added diff --git a/ofrak_core/pytest_ofrak/elf/assets/Makefile b/ofrak_core/pytest_ofrak/elf/assets/Makefile index d1123333b..121d60e88 100644 --- a/ofrak_core/pytest_ofrak/elf/assets/Makefile +++ b/ofrak_core/pytest_ofrak/elf/assets/Makefile @@ -1,4 +1,3 @@ - CC=gcc default: program diff --git a/ofrak_core/pytest_ofrak/elf/fixtures.py b/ofrak_core/pytest_ofrak/elf/fixtures.py index 99bbab873..32eedfeeb 100644 --- a/ofrak_core/pytest_ofrak/elf/fixtures.py +++ b/ofrak_core/pytest_ofrak/elf/fixtures.py @@ -1,133 +1,12 @@ import os -# import subprocess - import pytest -# MAKEFILE_CONTENTS = """ -# CC=gcc - -# default: program - -# program.o: program.c $(HEADERS) -# $(CC) -c program.c -fno-asynchronous-unwind-tables -o program.o - -# program: program.o -# $(CC) program.o -o program - -# program_no_reloc: program.o -# $(CC) program.o -no-pie -o program_no_reloc - -# program_relocated: program_relocated.o -# $(CC) program_relocated.o -o program_relocated - -# large_elf.o: large_elf.c $(HEADERS) -# $(CC) -c large_elf.c -o large_elf.o - -# large_elf: large_elf.o -# $(CC) large_elf.o -no-pie -o large_elf -# """ - -# C_SOURCE_CONTENTS = """ -# #include - -# int foo(); -# int bar(); - -# int main() { -# printf("Hello, World!\\n"); -# return foo(); -# } - -# int foo() { -# return 12; -# } - -# int bar() { -# return 24; -# } - - -# """ - -# LARGE_SOURCE_CONTENTS_HEADER = """ -# int foo(); -# int bar(); - -# int main() { -# return bar(); -# } - -# """ - -# LARGE_SOURCE_CONTENTS_FOOTER = """ - -# int foo() { -# return 12; -# } - -# int bar() { -# return 24; -# } - - -# """ - -# PATCH_CONTENTS = """ -# int noop0(); - -# int baz() -# { -# noop0(); -# return 36; -# } -# """ - - -# def create_noops(): -# noops = [] - -# i = 0 - -# for i in range(8000): -# instruction = f"int noop{i}(){{}}" -# noops.append(instruction) - -# return noops - - @pytest.fixture(scope="session") def elf_test_directory(): return os.path.join(os.path.dirname(__file__), "assets") - # makefile_path = os.path.join(tmpdir, "Makefile") - # c_source_path = os.path.join(tmpdir, "program.c") - # large_source_path = os.path.join(tmpdir, "large_elf.c") - - # patch_dir = os.path.join(tmpdir, "source_dir") - # if not os.path.exists(patch_dir): - # os.mkdir(patch_dir) - # patch_path = os.path.join(patch_dir, "patch.c") - - # noops = create_noops() - - # LARGE_SOURCE_CONTENTS = LARGE_SOURCE_CONTENTS_HEADER - # for noop in noops: - # LARGE_SOURCE_CONTENTS += noop - # LARGE_SOURCE_CONTENTS += LARGE_SOURCE_CONTENTS_FOOTER - - # with open(makefile_path, "w") as f: - # f.write(MAKEFILE_CONTENTS) - # with open(c_source_path, "w") as f: - # f.write(C_SOURCE_CONTENTS) - # with open(large_source_path, "w") as f: - # f.write(LARGE_SOURCE_CONTENTS) - # with open(patch_path, "w") as f: - # f.write(PATCH_CONTENTS) - - # return tmpdir - @pytest.fixture(scope="session") def elf_object_file(elf_test_directory): diff --git a/ofrak_core/requirements.txt b/ofrak_core/requirements.txt index 8fb088bae..f1d66e69f 100644 --- a/ofrak_core/requirements.txt +++ b/ofrak_core/requirements.txt @@ -10,8 +10,8 @@ importlib-metadata>=4.13 intervaltree==3.1.0 keystone-engine==0.9.2 jefferson==0.4.5;python_version>="3.8" -lief==0.15.1 -orjson~=3.9.15 +lief==0.16.1 +orjson~=3.10.12 pefile==2023.2.7 pycdlib==1.12.0 python-magic;platform_system!="Windows" diff --git a/ofrak_core/setup.py b/ofrak_core/setup.py index 8504489d0..ef27c580c 100644 --- a/ofrak_core/setup.py +++ b/ofrak_core/setup.py @@ -69,7 +69,7 @@ def read_requirements(requirements_path): setuptools.setup( name="ofrak", - version="3.3.0rc2", + version="3.3.0rc3", description="A binary analysis and modification platform", packages=setuptools.find_packages(exclude=["test_ofrak", "test_ofrak.*"]), package_data={ @@ -78,7 +78,7 @@ def read_requirements(requirements_path): install_requires=[ "ofrak_io>=1.0,==1.*", "ofrak_type>=2.2.0rc0,==2.*", - "ofrak_patch_maker>=4.0.2rc0,==4.*", + "ofrak_patch_maker>=4.1.0rc0,==4.*", ] + read_requirements("requirements.txt"), extras_require={ diff --git a/ofrak_patch_maker/CHANGELOG.md b/ofrak_patch_maker/CHANGELOG.md index f8ebe2230..7a7bb2a41 100644 --- a/ofrak_patch_maker/CHANGELOG.md +++ b/ofrak_patch_maker/CHANGELOG.md @@ -3,10 +3,10 @@ All notable changes to `ofrak-patch-maker` will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) +## [Unreleased: 4.1.0rc0](https://github.com/redballoonsecurity/ofrak/tree/master) ### Added - GNU 32-bit x86 toolchain. ([#405](https://github.com/redballoonsecurity/ofrak/pull/405)) -- SPARC BCC tolchain. ([#462](https://github.com/redballoonsecurity/ofrak/pull/462)) +- SPARC BCC toolchain (x86_64 platforms only) ([#462](https://github.com/redballoonsecurity/ofrak/pull/462), [#405](https://github.com/redballoonsecurity/ofrak/pull/405)) ### Removed - Switch from binutils version of linker, bin_parser, assembler for X86_64 to default Debian versions. ([#405](https://github.com/redballoonsecurity/ofrak/pull/405)) @@ -18,7 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Changed - Removed `SUBALIGN(0)` for `.bss` sections - Minor update to OFRAK Community License, add OFRAK Pro License ([#478](https://github.com/redballoonsecurity/ofrak/pull/478)) -- Install toolchains from source or tarball instead of relying on the Debian package manager ([#502](https://github.com/redballoonsecurity/ofrak/pull/502), [#541](https://github.com/redballoonsecurity/ofrak/pull/541)) +- Install toolchains (x86_64, aarch64 platforms) from source or tarball instead of relying on the Debian package manager ([#405](https://github.com/redballoonsecurity/ofrak/pull/405), [#502](https://github.com/redballoonsecurity/ofrak/pull/502), [#541](https://github.com/redballoonsecurity/ofrak/pull/541), [#562](https://github.com/redballoonsecurity/ofrak/pull/562/files)) - Deprecate reserving `.bss` space at the FEM linking step with `create_unsafe_bss_segment` method and `unsafe_bss_segment` argument. ([#505](https://github.com/redballoonsecurity/ofrak/pull/505)) - Determine the correct allocation behavior of sections from ELF flags rather than section names. ([#505](https://github.com/redballoonsecurity/ofrak/pull/505)) diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index cad28c3f8..c05d3d065 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -1,34 +1,53 @@ ARG TARGETARCH +RUN mkdir -p /opt/rbs/toolchain + +COPY ${PACKAGE_PATH}/ofrak_patch_maker/config/docker/${TARGETARCH}_platform_toolchain.conf /etc/ofrak/toolchain.conf + # LLVM -RUN mkdir -p /opt/rbs/toolchain && \ +RUN if [ "$TARGETARCH" = "amd64" ]; then \ cd /tmp && \ wget https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -C /opt/rbs/toolchain && \ rm -rf clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ - mv /opt/rbs/toolchain/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu- /opt/rbs/toolchain/llvm_12.0.1 + mv /opt/rbs/toolchain/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu- /opt/rbs/toolchain/llvm_12.0.1; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + apt-get update && apt-get install -y libtinfo5 && \ + cd /tmp/ && \ + wget https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/clang+llvm-12.0.1-aarch64-linux-gnu.tar.xz --show-progress --progress=bar:force:noscroll && \ + tar xf clang+llvm-12.0.1-aarch64-linux-gnu.tar.xz -C /opt/rbs/toolchain && \ + rm -rf clang+llvm-12.0.1-aarch64-linux-gnu.tar.xz && \ + mv /opt/rbs/toolchain/clang+llvm-12.0.1-aarch64-linux-gnu /opt/rbs/toolchain/llvm_12.0.1; \ +fi; # ARM GNU NONE EABI -RUN cd /tmp && \ +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + cd /tmp && \ wget https://developer.arm.com/-/media/Files/downloads/gnu-rm/10-2020q4/gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2 --show-progress --progress=bar:force:noscroll && \ tar xf gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2 -C /opt/rbs/toolchain && \ - rm -rf gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2 + rm -rf gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget https://developer.arm.com/-/media/Files/downloads/gnu-rm/10-2020q4/gcc-arm-none-eabi-10-2020-q4-major-aarch64-linux.tar.bz2 --show-progress --progress=bar:force:noscroll && \ + tar xf gcc-arm-none-eabi-10-2020-q4-major-aarch64-linux.tar.bz2 -C /opt/rbs/toolchain && \ + rm -rf gcc-arm-none-eabi-10-2020-q4-major-aarch64-linux.tar.bz2; \ +fi; # LINUX GNU + BINUTILS -RUN cd /tmp && \ +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + cd /tmp && \ wget https://download.01.org/0day-ci/cross-package/gcc-10.3.0-nolibc/x86_64-gcc-10.3.0-nolibc_x86_64-linux.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf x86_64-gcc-10.3.0-nolibc_x86_64-linux.tar.xz -C /opt/rbs/toolchain && \ - rm -rf x86_64-gcc-10.3.0-nolibc_x86_64-linux.tar.xz - -#X64-64 toolchain for arm64 Docker images -RUN if [ "$TARGETARCH" = "arm64" ]; then \ - apt-get update && apt-get install -y gcc-10-x86-64-linux-gnu; \ + rm -rf x86_64-gcc-10.3.0-nolibc_x86_64-linux.tar.xz; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget https://www.kernel.org/pub/tools/crosstool/files/bin/arm64/10.3.0/arm64-gcc-10.3.0-nolibc-x86_64-linux.tar.xz --show-progress --progress=bar:force:noscroll && \ + tar xvf arm64-gcc-10.3.0-nolibc-x86_64-linux.tar.xz -C /opt/rbs/toolchain && \ + rm -rf arm64-gcc-10.3.0-nolibc-x86_64-linux.tar.xz; \ fi; #M68k GNU 10 Linux -#Only exists for x86 -RUN if [ "$TARGETARCH" = "amd64" ]; then \ - cd /tmp && \ +RUN cd /tmp && \ apt-get update && apt-get install -y gcc g++ gperf bison flex texinfo help2man make libncurses5-dev python3-dev autoconf automake libtool libtool-bin gawk wget bzip2 xz-utils unzip patch libstdc++6 rsync && \ git clone https://github.com/crosstool-ng/crosstool-ng.git && \ cd crosstool-ng/ && \ @@ -82,8 +101,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ >> .config && \ ./ct-ng build CT_JOBS=`nproc` \ | sed 's/.*\r//g' && \ - cd /tmp && rm -rf crosstool-ng; \ -fi; + cd /tmp && rm -rf crosstool-ng #M68k VBCC RUN cd /tmp && \ @@ -100,7 +118,7 @@ RUN cd /tmp && \ CPU=m68k SYNTAX=mot make && \ cp ./vasmm68k_mot /opt/rbs/toolchain/vbcc_0_9/bin/ && \ cp ./vobjdump /opt/rbs/toolchain/vbcc_0_9/bin/ && \ - cd /tmp &&\ + cd /tmp && \ rm -rf vasm* vbcc* #AARCH64 GNU 10 Linux @@ -109,6 +127,11 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ wget https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz -C /opt/rbs/toolchain && \ rm -rf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget https://developer.arm.com/-/media/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz --show-progress --progress=bar:force:noscroll && \ + tar xf gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz -C /opt/rbs/toolchain && \ + rm -rf gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz; \ fi; #AVR GCC @@ -117,6 +140,11 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ wget https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/SoftwareLibraries/Firmware/avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz --show-progress --progress=bar:force:noscroll && \ tar xzf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz -C /opt/rbs/toolchain && \ rm -rf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget http://downloads.arduino.cc/tools/avr-gcc-7.3.0-atmel3.6.1-arduino7-aarch64-pc-linux-gnu.tar.bz2 --show-progress --progress=bar:force:noscroll && \ + tar xf avr-gcc-7.3.0-atmel3.6.1-arduino7-aarch64-pc-linux-gnu.tar.bz2 -C /opt/rbs/toolchain && \ + rm -rf avr-gcc-7.3.0-atmel3.6.1-arduino7-aarch64-pc-linux-gnu.tar.bz2; \ fi; #PPC GNU 10 Linux @@ -125,6 +153,11 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ wget https://download.01.org/0day-ci/cross-package/gcc-10.3.0-nolibc/x86_64-gcc-10.3.0-nolibc_powerpc-linux.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf x86_64-gcc-10.3.0-nolibc_powerpc-linux.tar.xz -C /opt/rbs/toolchain && \ rm -rf x86_64-gcc-10.3.0-nolibc_powerpc-linux.tar.xz; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget https://www.kernel.org/pub/tools/crosstool/files/bin/arm64/10.3.0/arm64-gcc-10.3.0-nolibc-powerpc-linux.tar.xz --show-progress --progress=bar:force:noscroll && \ + tar xf arm64-gcc-10.3.0-nolibc-powerpc-linux.tar.xz -C /opt/rbs/toolchain && \ + rm -rf arm64-gcc-10.3.0-nolibc-powerpc-linux.tar.xz; \ fi; #BCC (GCC) SPARC v8 diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/config/docker/amd64_platform_toolchain.conf similarity index 96% rename from ofrak_patch_maker/ofrak_patch_maker/toolchain.conf rename to ofrak_patch_maker/ofrak_patch_maker/config/docker/amd64_platform_toolchain.conf index 769dab9b9..410335511 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf +++ b/ofrak_patch_maker/ofrak_patch_maker/config/docker/amd64_platform_toolchain.conf @@ -26,7 +26,6 @@ LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump [GNU_X86_64_LINUX_EABI_10_3_0] -# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld @@ -34,7 +33,6 @@ BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux- LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib [GNU_X86_32_LINUX_EABI_10_3_0] -# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld diff --git a/ofrak_patch_maker/ofrak_patch_maker/config/docker/arm64_platform_toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/config/docker/arm64_platform_toolchain.conf new file mode 100644 index 000000000..1c3f4ad6d --- /dev/null +++ b/ofrak_patch_maker/ofrak_patch_maker/config/docker/arm64_platform_toolchain.conf @@ -0,0 +1,78 @@ +# Toolchain config for X86_64 Platforms. +[LLVM_12_0_1] +PREPROCESSOR = /opt/rbs/toolchain/llvm_12.0.1/bin/clang +COMPILER = /opt/rbs/toolchain/llvm_12.0.1/bin/clang +LINKER = /opt/rbs/toolchain/llvm_12.0.1/bin/ld.lld +BIN_PARSER = /opt/rbs/toolchain/llvm_12.0.1/bin/llvm-readobj +LIB = /opt/rbs/toolchain/llvm_12.0.1/lib + +[GNU_ARM_NONE_EABI_10_2_1] +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-objdump +LIB = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/lib + +[GNU_M68K_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump +LIB=/opt/rbs/toolchain/m68k-unknown-linux-gnu/lib + +[VBCC_M68K_0_9] +PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/vbcc_0_9/bin/vbccm68k +LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump + +[GNU_X86_64_LINUX_EABI_10_3_0] +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib + +[GNU_X86_32_LINUX_EABI_10_3_0] +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib + +[GNU_AARCH64_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-objdump +LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/lib + +[GNU_AVR_5] +PREPROCESSOR = /opt/rbs/toolchain/avr/bin/avr-gcc +COMPILER = /opt/rbs/toolchain/avr/bin/avr-gcc +LINKER = /opt/rbs/toolchain/avr/bin/avr-ld +BIN_PARSER = /opt/rbs/toolchain/avr/bin/avr-objdump +LIB = /opt/rbs/toolchain/avr/lib + +[GNU_PPC_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/lib + +[BCC_SPARC_GAISLER_ELF] +PREPROCESSOR = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc +COMPILER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc +LINKER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-ld +BIN_PARSER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-objdump +LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib + +[ASM] +ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as +X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as +M68K_ASM_PATH = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-as +AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-as +AVR_ASM_PATH = /opt/rbs/toolchain/avr/bin/avr-as +PPC_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-as +SPARC_ASM_PATH = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-as diff --git a/ofrak_patch_maker/ofrak_patch_maker/config/toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/config/toolchain.conf new file mode 100644 index 000000000..a2fd54135 --- /dev/null +++ b/ofrak_patch_maker/ofrak_patch_maker/config/toolchain.conf @@ -0,0 +1,81 @@ +# This file is the default toolchain config, to specify what toolchains ofrak_patch_maker uses. +# See "docker/amd64_platform_toolchain.conf" or "docker/arm64_platform_toolchain.conf" for examples +# of full toolchain configs used in OFRAK Docker images. +[LLVM_12_0_1] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[GNU_ARM_NONE_EABI_10_2_1] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[GNU_M68K_LINUX_10] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[VBCC_M68K_0_9] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[GNU_X86_64_LINUX_EABI_10_3_0] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[GNU_X86_32_LINUX_EABI_10_3_0] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[GNU_AARCH64_LINUX_10] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[GNU_AVR_5] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[GNU_PPC_LINUX_10] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[BCC_SPARC_GAISLER_ELF] +PREPROCESSOR = +COMPILER = +LINKER = +BIN_PARSER = +LIB = + +[ASM] +ARM_ASM_PATH = +X86_64_ASM_PATH = +M68K_ASM_PATH = +AARCH64_ASM_PATH = +AVR_ASM_PATH = +PPC_ASM_PATH = +SPARC_ASM_PATH = diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py index 0b53fc698..d13fe125d 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py @@ -12,7 +12,7 @@ from ofrak_type import ArchInfo from ofrak_patch_maker.binary_parser.abstract import AbstractBinaryFileParser from ofrak_patch_maker.toolchain.model import Segment, ToolchainConfig -from ofrak_patch_maker.toolchain.utils import get_repository_config +from ofrak_patch_maker.toolchain.utils import get_exec_from_config, get_repository_config from ofrak_type.architecture import InstructionSet from ofrak_type.bit_width import BitWidth from ofrak_type.memory_permissions import MemoryPermissions @@ -157,35 +157,35 @@ def _assembler_path(self) -> str: assembler_path = "SPARC_ASM_PATH" else: assembler_path = f"{self._processor.isa.value.upper()}_ASM_PATH" - return get_repository_config("ASM", assembler_path) + return get_exec_from_config("ASM", assembler_path) @property def _preprocessor_path(self) -> str: """ :return str: path to the toolchain preprocessor - this is usually the compiler. """ - return get_repository_config(self.name, "PREPROCESSOR") + return get_exec_from_config(self.name, "PREPROCESSOR") @property def _compiler_path(self) -> str: """ :return str: path to the toolchain compiler """ - return get_repository_config(self.name, "COMPILER") + return get_exec_from_config(self.name, "COMPILER") @property def _linker_path(self) -> str: """ :return str: path to the toolchain linker """ - return get_repository_config(self.name, "LINKER") + return get_exec_from_config(self.name, "LINKER") @property def _readobj_path(self) -> str: """ :return str: path to the toolchain binary analysis utility """ - return get_repository_config(self.name, "BIN_PARSER") + return get_exec_from_config(self.name, "BIN_PARSER") @property def _lib_path(self) -> str: diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py index dbcdf0ce0..cd597acdf 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py @@ -2,8 +2,8 @@ import os from multiprocessing import Pool, cpu_count from typing import Optional, Dict, Mapping, Tuple - import math +import shutil from ofrak_patch_maker.toolchain.model import BinFileType, Segment from ofrak_type.error import NotFoundError @@ -26,7 +26,8 @@ def get_file_format(path): def get_repository_config(section: str, key: Optional[str] = None): """ - Get config values from toolchain.conf. + Get config values from toolchain.conf. The toolchain.conf file can be located /etc/ofrak/toolchain.conf + or inside the `ofrak_patch_maker` Python package install. :param section: section name in config file :param key: key in `config[section]` @@ -38,11 +39,15 @@ def get_repository_config(section: str, key: Optional[str] = None): config = configparser.RawConfigParser() config_name = "toolchain.conf" - local_config = os.path.join(os.path.dirname(__file__), os.path.pardir) - config_paths = [local_config] + + # Should be shared across main OFRAK and use standardized configuration dirs in the future + config_paths = [ + os.path.join(p, config_name) + for p in ("/etc/ofrak/", os.path.join(os.path.dirname(__file__), os.path.pardir)) + ] + error_by_config_file: Dict[str, Exception] = dict() - for p in config_paths: - conf = os.path.join(p, config_name) + for conf in config_paths: if not os.path.exists(conf): continue try: @@ -57,7 +62,10 @@ def get_repository_config(section: str, key: Optional[str] = None): continue if 0 == len(error_by_config_file): - raise NotFoundError(f"Configuration file {config_name} not found") + config_paths_str = "\n".join(config_paths) + raise NotFoundError( + f"Configuration file {config_name} not found. Tried the following locations:\n{config_paths_str}" + ) elif 1 == len(error_by_config_file): _config, _e = next(iter(error_by_config_file.items())) @@ -70,6 +78,34 @@ def get_repository_config(section: str, key: Optional[str] = None): ) +def get_exec_from_config(section: str, key: str): + """ + Get an executable name or path from a toolchain.conf. Output instructions for configuring + toolchain paths if the exectuable is not available. + + :param section: section name in config file + :param key: key in `config[section]` + + :raises SystemExit: If `config[section]` or `config[section][key]` not found. + :return Union[str, List[Tuple[str, str]]]: the result of ``config.get(section, key)`` or + ``config.items(section)`` + """ + + exec_path = get_repository_config(section, key) + if shutil.which(exec_path) is not None: + return exec_path + else: + try: + download_link = get_repository_config(section, "DOWNLOAD") + download_text = f"download the toolchain from {download_link} and " + except NotFoundError: + download_text = "" + raise NotFoundError( + f"Configured executable {exec_path} ({key} of {section}) not found on the filesystem or PATH. " + f"Please {download_text}edit the toolchain.conf file to point to the toolchain executable." + ) + + # TODO: Add a main driver for this guy NULL_DATA = Segment( segment_name=".data", diff --git a/ofrak_patch_maker/ofrak_patch_maker_test/test_m68k_toolchain.py b/ofrak_patch_maker/ofrak_patch_maker_test/test_m68k_toolchain.py index 739d7568a..56aa15db5 100644 --- a/ofrak_patch_maker/ofrak_patch_maker_test/test_m68k_toolchain.py +++ b/ofrak_patch_maker/ofrak_patch_maker_test/test_m68k_toolchain.py @@ -3,7 +3,6 @@ import pytest -from ofrak_type import ArchInfo from ofrak_patch_maker.model import PatchRegionConfig from ofrak_patch_maker.patch_maker import PatchMaker from ofrak_patch_maker.toolchain.gnu_m68k import GNU_M68K_LINUX_10_Toolchain @@ -21,6 +20,7 @@ run_monkey_patch_test, ) from ofrak_patch_maker_test.toolchain_c import run_hello_world_test, run_bounds_check_test +from ofrak_type import ArchInfo from ofrak_type.architecture import ( InstructionSet, ProcessorType, diff --git a/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py b/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py index 3f2ebf178..7886623c3 100644 --- a/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py +++ b/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py @@ -1,4 +1,5 @@ import pytest +import platform from ofrak_patch_maker.toolchain.gnu_bcc_sparc import GNU_BCC_SPARC_Toolchain from ofrak_patch_maker_test import ToolchainUnderTest @@ -28,14 +29,17 @@ def toolchain_under_test(request) -> ToolchainUnderTest: return request.param +@pytest.mark.skipif(platform.machine() != "x86_64", reason="Test only supported on x86_64") def test_monkey_patch(toolchain_under_test: ToolchainUnderTest): run_monkey_patch_test(toolchain_under_test) # C Tests +@pytest.mark.skipif(platform.machine() != "x86_64", reason="Test only supported on x86_64") def test_bounds_check(toolchain_under_test: ToolchainUnderTest): run_bounds_check_test(toolchain_under_test) +@pytest.mark.skipif(platform.machine() != "x86_64", reason="Test only supported on x86_64") def test_hello_world(toolchain_under_test: ToolchainUnderTest): run_hello_world_test(toolchain_under_test) diff --git a/ofrak_patch_maker/setup.py b/ofrak_patch_maker/setup.py index d135259fb..2b81ab4fd 100644 --- a/ofrak_patch_maker/setup.py +++ b/ofrak_patch_maker/setup.py @@ -31,7 +31,7 @@ def read_requirements(requirements_path): setuptools.setup( name="ofrak_patch_maker", - version="4.0.2", + version="4.1.0rc0", description="PatchMaker tool for applying source-code patches to binaries", packages=setuptools.find_packages(exclude=("ofrak_patch_maker_test",)), package_data={"ofrak_patch_maker": ["py.typed"]},