diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..5990c6f --- /dev/null +++ b/.flake8 @@ -0,0 +1,8 @@ +[flake8] +select = E901,E999,F821,F822,F823 +max-line-length = 100 +exclude = .git,.eggs,__pycache__,build,dist,venv,third_party,_deps +max-complexity = 11 +show-source = true +statistics = true +count = true diff --git a/.github/workflows/scripts/build_nix.sh b/.github/workflows/scripts/build_nix.sh new file mode 100755 index 0000000..fa7adc2 --- /dev/null +++ b/.github/workflows/scripts/build_nix.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +pip install . diff --git a/.github/workflows/scripts/install_req_macos.sh b/.github/workflows/scripts/install_req_macos.sh new file mode 100755 index 0000000..1e27be7 --- /dev/null +++ b/.github/workflows/scripts/install_req_macos.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -e + +python -m pip install --upgrade pip +pip install -r requirements_dev.txt +brew install protobuf + +pip install setuptools wheel twine auditwheel diff --git a/.github/workflows/scripts/install_req_ubuntu.sh b/.github/workflows/scripts/install_req_ubuntu.sh new file mode 100755 index 0000000..e7d40f7 --- /dev/null +++ b/.github/workflows/scripts/install_req_ubuntu.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +sudo apt update -y +sudo apt install libboost-all-dev libprotobuf-dev protobuf-compiler curl git build-essential cmake automake libtool libtool-bin -y + +python -m pip install --upgrade pip +pip install -r requirements_dev.txt + +pip install setuptools wheel twine auditwheel diff --git a/.github/workflows/scripts/lint_cpp.sh b/.github/workflows/scripts/lint_cpp.sh new file mode 100755 index 0000000..e7f9245 --- /dev/null +++ b/.github/workflows/scripts/lint_cpp.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -e + +# Lint files (all .cpp and .h files) inplace. +find ./eva/ \( -iname *.h -o -iname *.cpp \) | xargs clang-format -i -style='file' +if [ $? -ne 0 ] +then + exit 1 +fi + +# Print changes. +git diff +# Already well formated if 'git diff' doesn't output anything. +! ( git diff | grep -q ^ ) || exit 1 diff --git a/.github/workflows/scripts/lint_python.sh b/.github/workflows/scripts/lint_python.sh new file mode 100755 index 0000000..d85e5b6 --- /dev/null +++ b/.github/workflows/scripts/lint_python.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -e + +# stop the build if there are Python syntax errors or undefined names +flake8 --config=.flake8 python/ +black --check python/ tests/ +# exit-zero treats all errors as warnings. +flake8 . --count --exit-zero --max-complexity=10 --max-line-length=100 --statistics diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..1f5af74 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,61 @@ +name: Tests + +on: + push: + branches: [main] + pull_request: + types: [opened, synchronize, reopened] + +jobs: + Linter: + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: [3.8] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: .github/workflows/scripts/install_req_ubuntu.sh + if: ${{ matrix.os == 'ubuntu-latest' }} + - name: Lint with flake8 + run: .github/workflows/scripts/lint_python.sh + - name: Lint with clang-format + run: .github/workflows/scripts/lint_cpp.sh + + Tests: + needs: [Linter] + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Prepare ENV + run: | + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV + if: ${{ matrix.os == 'ubuntu-latest' }} + - name: Install dependencies Ubuntu + run: .github/workflows/scripts/install_req_ubuntu.sh + if: ${{ matrix.os == 'ubuntu-latest' }} + - name: Install dependencies MacOS + run: .github/workflows/scripts/install_req_macos.sh + if: ${{ matrix.os == 'macos-latest' }} + - name: Build the library for Ubuntu/MacOS + run: .github/workflows/scripts/build_nix.sh + - name: Test + run: python tests/all.py diff --git a/.gitignore b/.gitignore index ecaea24..53ee1ae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -/.*/ /build/ /dist/ eva.egg-info/ @@ -16,4 +15,5 @@ CMakeFiles/ *.pb.h *.a *.so -*.whl \ No newline at end of file +*.whl +_deps diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 526cb48..0000000 --- a/.gitmodules +++ /dev/null @@ -1,8 +0,0 @@ -[submodule "third_party/pybind11"] - path = third_party/pybind11 - url = https://github.com/pybind/pybind11 - ignore = untracked -[submodule "third_party/Galois"] - path = third_party/Galois - url = https://github.com/IntelligentSoftwareSystems/Galois.git - ignore = untracked diff --git a/CMakeLists.txt b/CMakeLists.txt index 36cf734..a384315 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,19 +21,13 @@ if(USE_GALOIS) add_definitions(-DEVA_USE_GALOIS) endif() -find_package(SEAL 3.6 REQUIRED) find_package(Protobuf 3.6 REQUIRED) -find_package(Python COMPONENTS Interpreter Development) -if(NOT Python_VERSION_MAJOR EQUAL 3) - message(FATAL_ERROR "EVA requires Python 3. Please ensure you have it - installed in a location searched by CMake.") -endif() - -add_subdirectory(third_party/pybind11) +include(cmake/seal.cmake) +include(cmake/pybind11.cmake) if(USE_GALOIS) - add_subdirectory(third_party/Galois EXCLUDE_FROM_ALL) + include(cmake/galois.cmake) endif() -add_subdirectory(eva) -add_subdirectory(python) +include(cmake/eva.cmake) +include(cmake/python.cmake) diff --git a/README.md b/README.md index 666f8b3..73018e3 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ EVA is a native library written in C++17 with bindings for Python. Both Linux an To install dependencies on Ubuntu 20.04: ``` -sudo apt install cmake libboost-all-dev libprotobuf-dev protobuf-compiler +sudo apt install cmake libboost-all-dev libprotobuf-dev protobuf-compiler libfmt-dev ``` Clang is recommended for compilation, as SEAL is faster when compiled with it. To install clang and set it as default: @@ -25,36 +25,16 @@ sudo update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100 sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 ``` -Next install Microsoft SEAL version 3.6: -``` -git clone -b v3.6.4 https://github.com/microsoft/SEAL.git -cd SEAL -cmake -DSEAL_THROW_ON_TRANSPARENT_CIPHERTEXT=OFF . -make -j -sudo make install -``` -*Note that SEAL has to be installed with transparent ciphertext checking turned off, as it is not possible in general to statically ensure a program will not produce a transparent ciphertext. This does not affect the security of ciphertexts encrypted with SEAL.* - ### Building and Installing EVA #### Building EVA -EVA builds with CMake version ≥ 3.13: -``` -git submodule update --init -cmake . -make -j -``` -The build process creates a `setup.py` file in `python/`. To install the package for development with PIP: -``` -python3 -m pip install -e python/ -``` -To create a Python Wheel package for distribution in `dist/`: +To install the package with PIP: ``` -python3 python/setup.py bdist_wheel --dist-dir='.' +python3 -m pip install . ``` -To check that the installed Python package is working correctly, run all tests with: +To check that EVA is working correctly, run all tests with: ``` python3 tests/all.py ``` @@ -63,9 +43,9 @@ EVA does not yet support installing the native library for use in other CMake pr #### Multicore Support -EVA features highly scalable multicore support using the [Galois library](https://github.com/IntelligentSoftwareSystems/Galois). It is included as a submodule, but is turned off by default for faster builds and easier debugging. To build EVA with Galois configure with `USE_GALOIS=ON`: +EVA features highly scalable multicore support using the [Galois library](https://github.com/IntelligentSoftwareSystems/Galois), which is turned off by default for faster builds and easier debugging. To install EVA with multicore support use `setup.py` directly: ``` -cmake -DUSE_GALOIS=ON . +python3 setup.py install --use-galois ``` ### Running the Examples @@ -174,4 +154,4 @@ Many thanks to [Sangeeta Chowdhary](https://www.ilab.cs.rutgers.edu/~sc1696/), w Roshan Dathathri, Blagovesta Kostova, Olli Saarikivi, Wei Dai, Kim Laine, Madanlal Musuvathi. *EVA: An Encrypted Vector Arithmetic Language and Compiler for Efficient Homomorphic Computation*. PLDI 2020. [arXiv](https://arxiv.org/abs/1912.11951) [DOI](https://doi.org/10.1145/3385412.3386023) -Roshan Dathathri, Olli Saarikivi, Hao Chen, Kim Laine, Kristin Lauter, Saeed Maleki, Madanlal Musuvathi, Todd Mytkowicz. *CHET: An Optimizing Compiler for Fully-Homomorphic Neural-Network Inferencing*. PLDI 2019. [DOI](https://doi.org/10.1145/3314221.3314628) +Roshan Dathathri, Olli Saarikivi, Hao Chen, Kim Laine, Kristin Lauter, Saeed Maleki, Madanlal Musuvathi, Todd Mytkowicz. *CHET: An Optimizing Compiler for Fully-Homomorphic Neural-Network Inferencing*. PLDI 2019. [DOI](https://doi.org/10.1145/3314221.3314628) \ No newline at end of file diff --git a/eva/CMakeLists.txt b/cmake/eva.cmake similarity index 54% rename from eva/CMakeLists.txt rename to cmake/eva.cmake index bfee443..b100019 100644 --- a/eva/CMakeLists.txt +++ b/cmake/eva.cmake @@ -1,11 +1,31 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT license. + +set(EVA_SOURCES + eva/seal/seal.cpp + eva/util/logging.cpp + eva/common/reference_executor.cpp + eva/ckks/ckks_config.cpp -add_library(eva STATIC - eva.cpp - version.cpp + eva/ir/term.cpp + eva/ir/program.cpp + eva/ir/attribute_list.cpp + eva/ir/attributes.cpp + + eva/eva.cpp + eva/version.cpp ) +if(USE_GALOIS) + set(EVA_SOURCES ${EVA_SOURCES} + eva/util/galois.cpp + ) +endif() + +add_library(eva STATIC ${EVA_SOURCES}) + +add_subdirectory(${PROJECT_SOURCE_DIR}/eva/serialization/) + # TODO: everything except SEAL::seal should be make PRIVATE target_link_libraries(eva PUBLIC SEAL::seal protobuf::libprotobuf) if(USE_GALOIS) @@ -18,10 +38,3 @@ target_include_directories(eva $ ) target_compile_definitions(eva PRIVATE EVA_VERSION_STR="${PROJECT_VERSION}") - -add_subdirectory(util) -add_subdirectory(serialization) -add_subdirectory(ir) -add_subdirectory(common) -add_subdirectory(ckks) -add_subdirectory(seal) diff --git a/cmake/galois.cmake b/cmake/galois.cmake new file mode 100644 index 0000000..0c751f3 --- /dev/null +++ b/cmake/galois.cmake @@ -0,0 +1,16 @@ +include(FetchContent) + +FetchContent_Declare( + com_intelligentsoftwaresystems_galois + GIT_REPOSITORY https://github.com/IntelligentSoftwareSystems/Galois + GIT_TAG release-6.0 +) +FetchContent_MakeAvailable(com_intelligentsoftwaresystems_galois) + +if(IS_DIRECTORY "${com_intelligentsoftwaresystems_galois_SOURCE_DIR}") + set_property(DIRECTORY ${com_intelligentsoftwaresystems_galois_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL YES) +endif() + +include_directories( + ${com_intelligentsoftwaresystems_galois_SOURCE_DIR}/include +) \ No newline at end of file diff --git a/cmake/pybind11.cmake b/cmake/pybind11.cmake new file mode 100644 index 0000000..9057643 --- /dev/null +++ b/cmake/pybind11.cmake @@ -0,0 +1,12 @@ +include(FetchContent) + +FetchContent_Declare( + com_pybind_pybind11 + GIT_REPOSITORY https://github.com/pybind/pybind11 + GIT_TAG v2.6.2 +) +FetchContent_MakeAvailable(com_pybind_pybind11) + +include_directories( + ${com_pybind_pybind11_SOURCE_DIR}/include +) diff --git a/cmake/python.cmake b/cmake/python.cmake new file mode 100644 index 0000000..5125958 --- /dev/null +++ b/cmake/python.cmake @@ -0,0 +1,13 @@ + # Copyright (c) Microsoft Corporation. All rights reserved. + # Licensed under the MIT license. + + pybind11_add_module(eva_py + python/eva/wrapper.cpp + ) + + target_compile_features(eva_py PUBLIC cxx_std_17) + target_link_libraries(eva_py PRIVATE eva) + if (MSVC) + target_link_libraries(eva_py PUBLIC bcrypt) + endif() + set_target_properties(eva_py PROPERTIES OUTPUT_NAME _eva) diff --git a/cmake/seal.cmake b/cmake/seal.cmake new file mode 100644 index 0000000..bb53580 --- /dev/null +++ b/cmake/seal.cmake @@ -0,0 +1,17 @@ +include(FetchContent) + +set(SEAL_BUILD_DEPS ON) +set(SEAL_USE_MSGSL ON) +option(SEAL_THROW_ON_TRANSPARENT_CIPHERTEXT OFF) + +FetchContent_Declare( + com_microsoft_seal + GIT_REPOSITORY https://github.com/microsoft/SEAL + GIT_TAG v3.6.4 +) +FetchContent_MakeAvailable(com_microsoft_seal) + +include_directories(${com_microsoft_seal_SOURCE_DIR}/native/src) +include_directories(${com_microsoft_seal_SOURCE_DIR}/thirdparty/msgsl-src/include/) +include_directories(${com_microsoft_seal_SOURCE_DIR}/thirdparty/hexl-src/hexl/include/) +include_directories(${com_microsoft_seal_BINARY_DIR}/native/src) diff --git a/eva/ckks/CMakeLists.txt b/eva/ckks/CMakeLists.txt deleted file mode 100644 index 3c325a4..0000000 --- a/eva/ckks/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT license. - -target_sources(eva PRIVATE - ckks_config.cpp -) diff --git a/eva/ckks/mod_switcher.h b/eva/ckks/mod_switcher.h index 2fb7dfc..3ac6d9c 100644 --- a/eva/ckks/mod_switcher.h +++ b/eva/ckks/mod_switcher.h @@ -54,7 +54,7 @@ class ModSwitcher { Term::Ptr &term) { // must only be used with backward pass traversal if (term->numUses() == 0) return; - //we do not want to add modswitch for nodes of type raw + // we do not want to add modswitch for nodes of type raw if (type[term] == Type::Raw) return; if (term->op == Op::Encode) { diff --git a/eva/common/CMakeLists.txt b/eva/common/CMakeLists.txt deleted file mode 100644 index 07e8197..0000000 --- a/eva/common/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT license. - -target_sources(eva PRIVATE - reference_executor.cpp -) diff --git a/eva/ir/CMakeLists.txt b/eva/ir/CMakeLists.txt deleted file mode 100644 index c54617b..0000000 --- a/eva/ir/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT license. - -target_sources(eva PRIVATE - term.cpp - program.cpp - attribute_list.cpp - attributes.cpp -) diff --git a/eva/util/CMakeLists.txt b/eva/util/CMakeLists.txt deleted file mode 100644 index 55f959e..0000000 --- a/eva/util/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT license. - -if(USE_GALOIS) - target_sources(eva PRIVATE - galois.cpp - ) -endif() - -target_sources(eva PRIVATE - logging.cpp -) diff --git a/examples/image_processing.py b/examples/image_processing.py index b437b71..91d245a 100644 --- a/examples/image_processing.py +++ b/examples/image_processing.py @@ -8,6 +8,7 @@ from PIL import Image import numpy as np + def convolution(image, width, filter): for i in range(len(filter)): for j in range(len(filter[0])): @@ -19,6 +20,7 @@ def convolution(image, width, filter): convolved += partial return convolved + def convolutionXY(image, width, filter): for i in range(len(filter)): for j in range(len(filter[0])): @@ -33,17 +35,15 @@ def convolutionXY(image, width, filter): Iy += vertical return Ix, Iy + h = 64 w = 64 -sobel = EvaProgram('sobel', vec_size=h*w) +sobel = EvaProgram("sobel", vec_size=h * w) with sobel: - image = Input('image') + image = Input("image") - sobel_filter = [ - [-1, 0, 1], - [-2, 0, 2], - [-1, 0, 1]] + sobel_filter = [[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]] a1 = 2.2137874823876622 a2 = -1.0984324107372518 @@ -51,39 +51,33 @@ def convolutionXY(image, width, filter): conv_hor, conv_ver = convolutionXY(image, w, sobel_filter) - conv_hor2 = conv_hor**2 - conv_ver2 = conv_ver**2 + conv_hor2 = conv_hor ** 2 + conv_ver2 = conv_ver ** 2 dsq = conv_hor2 + conv_ver2 dsq2 = dsq * dsq dsq3 = dsq2 * dsq - Output('image', dsq * a1 + dsq2 * a2 + dsq3 * a3) + Output("image", dsq * a1 + dsq2 * a2 + dsq3 * a3) sobel.set_input_scales(25) sobel.set_output_ranges(10) -harris = EvaProgram('harris', vec_size=h*w) +harris = EvaProgram("harris", vec_size=h * w) with harris: - image = Input('image') + image = Input("image") - sobel_filter = [ - [-1, 0, 1], - [-2, 0, 2], - [-1, 0, 1]] - pool = [ - [1, 1, 1], - [1, 1, 1], - [1, 1, 1]] + sobel_filter = [[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]] + pool = [[1, 1, 1], [1, 1, 1], [1, 1, 1]] c = 0.04 Ix, Iy = convolutionXY(image, w, sobel_filter) - - Ixx = Ix**2 - Iyy = Iy**2 + + Ixx = Ix ** 2 + Iyy = Iy ** 2 Ixy = Ix * Iy - #FIX: masking may be needed here (to handle boundaries) + # FIX: masking may be needed here (to handle boundaries) Sxx = convolution(Ixx, w, pool) Syy = convolution(Iyy, w, pool) @@ -94,26 +88,29 @@ def convolutionXY(image, width, filter): det = SxxSyy - SxySxy trace = Sxx + Syy - Output('image', det - trace**2 * c) + Output("image", det - trace ** 2 * c) harris.set_input_scales(30) harris.set_output_ranges(20) + def read_input_image(): - image = Image.open('baboon.png').convert('L') + image = Image.open("baboon.png").convert("L") image_data = [x / 255.0 for x in list(image.getdata())] - return {'image': image_data} + return {"image": image_data} + def write_output_image(outputs, tag): - enc_result_image = Image.new('L', (w, h)) - enc_result_image.putdata([x * 255.0 for x in outputs['image'][0:h*w]]) - enc_result_image.save(f'baboon_{tag}.png', "PNG") + enc_result_image = Image.new("L", (w, h)) + enc_result_image.putdata([x * 255.0 for x in outputs["image"][0 : h * w]]) + enc_result_image.save(f"baboon_{tag}.png", "PNG") + if __name__ == "__main__": inputs = read_input_image() for prog in [sobel, harris]: - print(f'Compiling {prog.name}') + print(f"Compiling {prog.name}") compiler = CKKSCompiler() compiled, params, signature = compiler.compile(prog) @@ -122,10 +119,10 @@ def write_output_image(outputs, tag): enc_outputs = public_ctx.execute(compiled, enc_inputs) outputs = secret_ctx.decrypt(enc_outputs, signature) - write_output_image(outputs, f'{compiled.name}_encrypted') + write_output_image(outputs, f"{compiled.name}_encrypted") reference = evaluate(compiled, inputs) - write_output_image(reference, f'{compiled.name}_reference') - - print('MSE', valuation_mse(outputs, reference)) + write_output_image(reference, f"{compiled.name}_reference") + + print("MSE", valuation_mse(outputs, reference)) print() diff --git a/examples/serialization.py b/examples/serialization.py index 11a4c2b..7a01178 100644 --- a/examples/serialization.py +++ b/examples/serialization.py @@ -8,12 +8,12 @@ import numpy as np ################################################# -print('Compile time') +print("Compile time") -poly = EvaProgram('Polynomial', vec_size=8) +poly = EvaProgram("Polynomial", vec_size=8) with poly: - x = Input('x') - Output('y', 3*x**2 + 5*x - 2) + x = Input("x") + Output("y", 3 * x ** 2 + 5 * x - 2) poly.set_output_ranges(20) poly.set_input_scales(20) @@ -21,53 +21,51 @@ compiler = CKKSCompiler() poly, params, signature = compiler.compile(poly) -save(poly, 'poly.eva') -save(params, 'poly.evaparams') -save(signature, 'poly.evasignature') +save(poly, "poly.eva") +save(params, "poly.evaparams") +save(signature, "poly.evasignature") ################################################# -print('Key generation time') +print("Key generation time") -params = load('poly.evaparams') +params = load("poly.evaparams") public_ctx, secret_ctx = generate_keys(params) -save(public_ctx, 'poly.sealpublic') -save(secret_ctx, 'poly.sealsecret') +save(public_ctx, "poly.sealpublic") +save(secret_ctx, "poly.sealsecret") ################################################# -print('Runtime on client') +print("Runtime on client") -signature = load('poly.evasignature') -public_ctx = load('poly.sealpublic') +signature = load("poly.evasignature") +public_ctx = load("poly.sealpublic") -inputs = { - 'x': [i for i in range(signature.vec_size)] -} +inputs = {"x": [i for i in range(signature.vec_size)]} encInputs = public_ctx.encrypt(inputs, signature) -save(encInputs, 'poly_inputs.sealvals') +save(encInputs, "poly_inputs.sealvals") ################################################# -print('Runtime on server') +print("Runtime on server") -poly = load('poly.eva') -public_ctx = load('poly.sealpublic') -encInputs = load('poly_inputs.sealvals') +poly = load("poly.eva") +public_ctx = load("poly.sealpublic") +encInputs = load("poly_inputs.sealvals") encOutputs = public_ctx.execute(poly, encInputs) -save(encOutputs, 'poly_outputs.sealvals') +save(encOutputs, "poly_outputs.sealvals") ################################################# -print('Back on client') +print("Back on client") -secret_ctx = load('poly.sealsecret') -encOutputs = load('poly_outputs.sealvals') +secret_ctx = load("poly.sealsecret") +encOutputs = load("poly_outputs.sealvals") outputs = secret_ctx.decrypt(encOutputs, signature) reference = evaluate(poly, inputs) -print('Expected', reference) -print('Got', outputs) -print('MSE', valuation_mse(outputs, reference)) +print("Expected", reference) +print("Got", outputs) +print("MSE", valuation_mse(outputs, reference)) diff --git a/python/.gitignore b/python/.gitignore deleted file mode 100644 index fca8a26..0000000 --- a/python/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# In-source build files -setup.py -setup.py.after_configure \ No newline at end of file diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt deleted file mode 100644 index af91c62..0000000 --- a/python/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT license. - -configure_file(setup.py.in setup.py.after_configure) -file(GENERATE OUTPUT setup.py INPUT ${CMAKE_CURRENT_BINARY_DIR}/setup.py.after_configure) - -add_subdirectory(eva) - -add_custom_target(python ALL DEPENDS eva_py) -add_custom_command(TARGET python - POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_directory - ${CMAKE_CURRENT_SOURCE_DIR}/eva - ${CMAKE_CURRENT_BINARY_DIR}/eva -) diff --git a/python/eva/CMakeLists.txt b/python/eva/CMakeLists.txt deleted file mode 100644 index 1b292aa..0000000 --- a/python/eva/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT license. - -pybind11_add_module(eva_py - wrapper.cpp -) - -target_compile_features(eva_py PUBLIC cxx_std_17) -target_link_libraries(eva_py PRIVATE eva) -if (MSVC) - target_link_libraries(eva_py PUBLIC bcrypt) -endif() -set_target_properties(eva_py PROPERTIES OUTPUT_NAME _eva) diff --git a/python/eva/__init__.py b/python/eva/__init__.py index 8c0a211..7d55a38 100644 --- a/python/eva/__init__.py +++ b/python/eva/__init__.py @@ -1,7 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT license. -from ._eva import * +from _eva import * +import _eva import numbers import psutil @@ -14,13 +15,16 @@ set_num_threads(_default_num_threads) _current_program = None + + def _curr(): """ Returns the EvaProgram that is currently in context """ global _current_program - if _current_program == None: + if _current_program is None: raise RuntimeError("No Program in context") return _current_program + def _py_to_term(x, program): """ Maps supported types into EVA terms """ if isinstance(x, Expr): @@ -34,11 +38,12 @@ def _py_to_term(x, program): else: raise TypeError("No conversion to Term available for " + str(x)) -def py_to_eva(x, program = None): + +def py_to_eva(x, program=None): """ Maps supported types into EVA terms. May be used in library functions - to provide uniform support for Expr instances and python types that + to provide uniform support for Expr instances and python types that are convertible into constants in EVA programs. - + Parameters ---------- x : eva.Expr, EVA native Term, list or a number @@ -50,14 +55,15 @@ def py_to_eva(x, program = None): if isinstance(x, Expr): return x else: - if program == None: + if program is None: program = _curr() return Expr(_py_to_term(x, program), program) -class Expr(): + +class Expr: """ Wrapper for EVA's native Term class. Provides operator overloads that create terms in the associated EvaProgram. - + Attributes ---------- term @@ -70,58 +76,91 @@ def __init__(self, term, program): self.term = term self.program = program - def __add__(self,other): + def __add__(self, other): """ Create a new addition term """ - return Expr(self.program._make_term(Op.Add, [self.term, _py_to_term(other, self.program)]), self.program) - - def __radd__(self,other): + return Expr( + self.program._make_term( + Op.Add, [self.term, _py_to_term(other, self.program)] + ), + self.program, + ) + + def __radd__(self, other): """ Create a new addition term """ - return Expr(self.program._make_term(Op.Add, [_py_to_term(other, self.program), self.term]), self.program) - - def __sub__(self,other): + return Expr( + self.program._make_term( + Op.Add, [_py_to_term(other, self.program), self.term] + ), + self.program, + ) + + def __sub__(self, other): """ Create a new subtraction term """ - return Expr(self.program._make_term(Op.Sub, [self.term, _py_to_term(other, self.program)]), self.program) - - def __rsub__(self,other): + return Expr( + self.program._make_term( + Op.Sub, [self.term, _py_to_term(other, self.program)] + ), + self.program, + ) + + def __rsub__(self, other): """ Create a new subtraction term """ - return Expr(self.program._make_term(Op.Sub, [_py_to_term(other, self.program), self.term]), self.program) - - def __mul__(self,other): + return Expr( + self.program._make_term( + Op.Sub, [_py_to_term(other, self.program), self.term] + ), + self.program, + ) + + def __mul__(self, other): """ Create a new multiplication term """ - return Expr(self.program._make_term(Op.Mul, [self.term, _py_to_term(other, self.program)]), self.program) - - def __rmul__(self,other): + return Expr( + self.program._make_term( + Op.Mul, [self.term, _py_to_term(other, self.program)] + ), + self.program, + ) + + def __rmul__(self, other): """ Create a new multiplication term """ - return Expr(self.program._make_term(Op.Mul, [_py_to_term(other, self.program), self.term]), self.program) - - def __pow__(self,exponent): + return Expr( + self.program._make_term( + Op.Mul, [_py_to_term(other, self.program), self.term] + ), + self.program, + ) + + def __pow__(self, exponent): """ Create exponentiation as nested multiplication terms """ if exponent < 1: raise ValueError("exponent must be greater than zero, got " + exponent) result = self.term - for i in range(exponent-1): + for i in range(exponent - 1): result = self.program._make_term(Op.Mul, [result, self.term]) return Expr(result, self.program) - def __lshift__(self,rotation): + def __lshift__(self, rotation): """ Create a left rotation term """ return Expr(self.program._make_left_rotation(self.term, rotation), self.program) - def __rshift__(self,rotation): + def __rshift__(self, rotation): """ Create a right rotation term """ - return Expr(self.program._make_right_rotation(self.term, rotation), self.program) + return Expr( + self.program._make_right_rotation(self.term, rotation), self.program + ) def __neg__(self): """ Create a negation term """ return Expr(self.program._make_term(Op.Negate, [self.term]), self.program) + class EvaProgram(Program): """ A wrapper for EVA's native Program class. Acts as a context manager to set the program the Input and Output free functions operate on. """ def __init__(self, name, vec_size): """ Create a new EvaProgram with a name and a vector size - + Parameters ---------- name : str @@ -134,16 +173,17 @@ def __init__(self, name, vec_size): def __enter__(self): global _current_program - if _current_program != None: + if _current_program is not None: raise RuntimeError("There is already an EVA Program in context") _current_program = self - + def __exit__(self, exc_type, exc_value, exc_traceback): global _current_program if _current_program != self: raise RuntimeError("This program is not currently in context") _current_program = None + def Input(name, is_encrypted=True): """ Create a new named input term in the current EvaProgram @@ -155,7 +195,10 @@ def Input(name, is_encrypted=True): Whether this input should be encrypted or not (default: True) """ program = _curr() - return Expr(program._make_input(name, Type.Cipher if is_encrypted else Type.Raw), program) + return Expr( + program._make_input(name, Type.Cipher if is_encrypted else Type.Raw), program + ) + def Output(name, expr): """ Create a new named output term in the current EvaProgram diff --git a/python/eva/metric.py b/python/eva/metric.py index f265ea9..b4c670c 100644 --- a/python/eva/metric.py +++ b/python/eva/metric.py @@ -3,7 +3,8 @@ import numpy as _np -def valuation_mse(a,b): + +def valuation_mse(a, b): """ Calculate the total mean squared error between two valuations Parameters @@ -15,5 +16,5 @@ def valuation_mse(a,b): raise ValueError("Valuations must have the same keys") mse = 0 for k in a.keys(): - mse += _np.mean((_np.array(a[k]) - _np.array(b[k]))**2) + mse += _np.mean((_np.array(a[k]) - _np.array(b[k])) ** 2) return mse / len(a) diff --git a/eva/seal/CMakeLists.txt b/python/eva/std/__init__.py similarity index 69% rename from eva/seal/CMakeLists.txt rename to python/eva/std/__init__.py index 005051a..fd51d79 100644 --- a/eva/seal/CMakeLists.txt +++ b/python/eva/std/__init__.py @@ -1,6 +1,4 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT license. -target_sources(eva PRIVATE - seal.cpp -) +from eva.std.numeric import * diff --git a/python/eva/std/numeric.py b/python/eva/std/numeric.py index ba4cb5b..340e1d9 100644 --- a/python/eva/std/numeric.py +++ b/python/eva/std/numeric.py @@ -2,20 +2,21 @@ # Licensed under the MIT license. from eva import py_to_eva + def horizontal_sum(x): - """ Sum together all elements of a vector. The result is replicated in all - elements of the returned vector. - - Parameters - ---------- - x : an EVA compatible type (see eva.py_to_eva) - The vector to sum together - """ + """ Sum together all elements of a vector. The result is replicated in all + elements of the returned vector. + + Parameters + ---------- + x : an EVA compatible type (see eva.py_to_eva) + The vector to sum together + """ - x = py_to_eva(x) - i = 1 - while i < x.program.vec_size: - y = x << i - x = x + y - i <<= 1 - return x + x = py_to_eva(x) + i = 1 + while i < x.program.vec_size: + y = x << i + x = x + y + i <<= 1 + return x diff --git a/python/setup.py.in b/python/setup.py.in deleted file mode 100644 index 270c1d2..0000000 --- a/python/setup.py.in +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT license. - -from setuptools import setup, find_packages -from setuptools.dist import Distribution - -class BinaryDistribution(Distribution): - """Distribution which always forces a binary package with platform name""" - def has_ext_modules(foo): - return True - -setup( - name='eva', - version='${PROJECT_VERSION}', - author='Microsoft Research EVA compiler team', - author_email='evacompiler@microsoft.com', - description='Compiler for the Microsoft SEAL homomorphic encryption library', - packages=find_packages('${CMAKE_CURRENT_BINARY_DIR}'), - package_data={ - 'eva': ['$'], - }, - distclass=BinaryDistribution, - install_requires=[ - 'psutil', - ], -) diff --git a/python/version.py b/python/version.py new file mode 100644 index 0000000..5c4105c --- /dev/null +++ b/python/version.py @@ -0,0 +1 @@ +__version__ = "1.0.1" diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..36785df --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,3 @@ +black==19.10b0 +flake8 +numpy diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e5f896b --- /dev/null +++ b/setup.py @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT license. + +import os +import sys +import platform +import subprocess +import re + + +from setuptools import setup, find_packages +from setuptools.dist import Distribution +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext +from setuptools.command.install import install +from setuptools.command.develop import develop +from distutils.version import LooseVersion + + +class BinaryDistribution(Distribution): + """Distribution which always forces a binary package with platform name""" + + def has_ext_modules(foo): + return True + + +def read(fname): + return open(os.path.join(os.path.dirname(__file__), fname)).read() + + +def find_version(): + version_file = read("python/version.py") + version_re = r"__version__ = \"(?P.+)\"" + version = re.match(version_re, version_file).group("version") + return version + + +class CMakeExtension(Extension): + def __init__(self, name, sourcedir=""): + Extension.__init__(self, name, sources=[]) + self.sourcedir = os.path.abspath(sourcedir) + self.user_cmake_args = [] + + +class CMakeBuild(build_ext): + def run(self): + try: + out = subprocess.check_output(["cmake", "--version"]) + except OSError: + raise RuntimeError( + "CMake must be installed to build the following extensions: " + + ", ".join(e.name for e in self.extensions) + ) + + if platform.system() == "Windows": + cmake_version = LooseVersion( + re.search(r"version\s*([\d.]+)", out.decode()).group(1) + ) + if cmake_version < "3.1.0": + raise RuntimeError("CMake >= 3.1.0 is required on Windows") + + for ext in self.extensions: + self.build_extension(ext) + + def build_extension(self, ext): + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + cmake_args = [ + "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=" + extdir, + "-DPYTHON_EXECUTABLE=" + sys.executable, + ] + + cfg = "Debug" if self.debug else "Release" + build_args = ["--config", cfg] + + if platform.system() == "Windows": + cmake_args += [f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"] + if sys.maxsize > 2 ** 32: + cmake_args += ["-A", "x64"] + build_args += ["--", "/m"] + else: + cmake_args += ["-DCMAKE_BUILD_TYPE=" + cfg] + build_args += ["--", "-j"] + + env = os.environ.copy() + env["CXXFLAGS"] = '{} -DVERSION_INFO=\\"{}\\"'.format( + env.get("CXXFLAGS", ""), self.distribution.get_version() + ) + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + subprocess.check_call( + ["cmake", ext.sourcedir] + cmake_args + ext.user_cmake_args, cwd=self.build_temp, env=env + ) + subprocess.check_call( + ["cmake", "--build", "."] + build_args, cwd=self.build_temp + ) + + +def add_eva_options(cls): + class eva_cls(cls): + user_options = cls.user_options + [("use-galois", None, "Use the Galois library for multicore homomorphic evaluation")] + + def initialize_options(self): + cls.initialize_options(self) + self.use_galois = False + + def finalize_options(self): + cls.finalize_options(self) + for ext in self.distribution.ext_modules: + if(ext.name == "eva_py"): + ext.user_cmake_args.append("-UUSE_GALOIS") + if self.use_galois: + ext.user_cmake_args.append("-DUSE_GALOIS=ON") + + def run(self): + cls.run(self) + return eva_cls + + +setup( + name="eva", + version=find_version(), + author="Microsoft Research EVA compiler team", + author_email="evacompiler@microsoft.com", + description="Compiler for the Microsoft SEAL homomorphic encryption library", + long_description=read("README.md"), + long_description_content_type="text/markdown", + packages=["eva", "eva.std", "eva.ckks", "eva.seal"], + package_dir={"": "python"}, + ext_modules=[CMakeExtension("eva_py")], + cmdclass=dict(build_ext=CMakeBuild, install=add_eva_options(install), develop=add_eva_options(develop)), + zip_safe=False, + install_requires=[ + 'psutil', + ], +) \ No newline at end of file diff --git a/tests/all.py b/tests/all.py index 46f2615..885c96c 100644 --- a/tests/all.py +++ b/tests/all.py @@ -6,5 +6,5 @@ from large_programs import * from std import * -if __name__ == '__main__': - unittest.main() \ No newline at end of file +if __name__ == "__main__": + unittest.main() diff --git a/tests/bug_fixes.py b/tests/bug_fixes.py index 292401f..110beff 100644 --- a/tests/bug_fixes.py +++ b/tests/bug_fixes.py @@ -5,8 +5,8 @@ from common import * from eva import EvaProgram, Input, Output -class BugFixes(EvaTestCase): +class BugFixes(EvaTestCase): def test_high_inner_term_scale(self): """ Test lazy waterline rescaler with a program causing a high inner term scale @@ -14,18 +14,20 @@ def test_high_inner_term_scale(self): rescaling not being inserted (causing high scales to be accumulated) and parameter selection not handling high scales in inner terms.""" - prog = EvaProgram('HighInnerTermScale', vec_size=4) + prog = EvaProgram("HighInnerTermScale", vec_size=4) with prog: - x1 = Input('x1') - x2 = Input('x2') - Output('y', x1*x1*x2) + x1 = Input("x1") + x2 = Input("x2") + Output("y", x1 * x1 * x2) prog.set_output_ranges(20) prog.set_input_scales(60) - - self.assert_compiles_and_matches_reference(prog, config={'rescaler':'lazy_waterline'}) - @unittest.skip('not fixed in SEAL yet') + self.assert_compiles_and_matches_reference( + prog, config={"rescaler": "lazy_waterline"} + ) + + @unittest.skip("not fixed in SEAL yet") def test_large_and_small(self): """ Check that a ciphertext with very large and small values decodes accurately @@ -34,19 +36,19 @@ def test_large_and_small(self): provide good accuracy for small values in ciphertexts when other very large values are present.""" - prog = EvaProgram('LargeAndSmall', vec_size=4) + prog = EvaProgram("LargeAndSmall", vec_size=4) with prog: - x = Input('x') - Output('y', pow(x,8)) + x = Input("x") + Output("y", pow(x, 8)) prog.set_output_ranges(60) prog.set_input_scales(60) - inputs = { - 'x': [0,1,10,100] - } + inputs = {"x": [0, 1, 10, 100]} - self.assert_compiles_and_matches_reference(prog, inputs, config={'warn_vec_size':'false'}) + self.assert_compiles_and_matches_reference( + prog, inputs, config={"warn_vec_size": "false"} + ) def test_output_rescaled(self): """ Check that the lazy waterline policy rescales outputs @@ -55,17 +57,20 @@ def test_output_rescaled(self): more primes in their modulus than necessary, which causes them to take more space when serialized.""" - prog = EvaProgram('OutputRescaled', vec_size=4) + prog = EvaProgram("OutputRescaled", vec_size=4) with prog: - x = Input('x') - Output('y', x*x) + x = Input("x") + Output("y", x * x) prog.set_output_ranges(20) prog.set_input_scales(60) - compiler = CKKSCompiler(config={'rescaler':'lazy_waterline', 'warn_vec_size':'false'}) + compiler = CKKSCompiler( + config={"rescaler": "lazy_waterline", "warn_vec_size": "false"} + ) prog, params, signature = compiler.compile(prog) self.assertEqual(params.prime_bits, [60, 20, 60, 60]) -if __name__ == '__main__': + +if __name__ == "__main__": unittest.main() diff --git a/tests/common.py b/tests/common.py index 3882a67..ed2e997 100644 --- a/tests/common.py +++ b/tests/common.py @@ -8,22 +8,24 @@ from eva.seal import generate_keys from eva.metric import valuation_mse + class EvaTestCase(unittest.TestCase): - def assert_compiles_and_matches_reference(self, prog, inputs = None, config={}): + def assert_compiles_and_matches_reference(self, prog, inputs=None, config={}): if inputs == None: - inputs = { name: [uniform(-2,2) for _ in range(prog.vec_size)] - for name in prog.inputs } - config['warn_vec_size'] = 'false' + inputs = { + name: [uniform(-2, 2) for _ in range(prog.vec_size)] + for name in prog.inputs + } + config["warn_vec_size"] = "false" reference = evaluate(prog, inputs) - compiler = CKKSCompiler(config = config) + compiler = CKKSCompiler(config=config) compiled_prog, params, signature = compiler.compile(prog) reference_compiled = evaluate(compiled_prog, inputs) ref_mse = valuation_mse(reference, reference_compiled) - self.assertTrue(ref_mse < 0.0000000001, - f"Mean squared error was {ref_mse}") + self.assertTrue(ref_mse < 0.0000000001, f"Mean squared error was {ref_mse}") public_ctx, secret_ctx = generate_keys(params) encInputs = public_ctx.encrypt(inputs, signature) @@ -33,4 +35,4 @@ def assert_compiles_and_matches_reference(self, prog, inputs = None, config={}): he_mse = valuation_mse(outputs, reference) self.assertTrue(he_mse < 0.01, f"Mean squared error was {he_mse}") - return (compiled_prog, params, signature) \ No newline at end of file + return (compiled_prog, params, signature) diff --git a/tests/features.py b/tests/features.py index 843e5a0..73c9931 100644 --- a/tests/features.py +++ b/tests/features.py @@ -7,6 +7,7 @@ from common import * from eva import EvaProgram, Input, Output, save, load + class Features(EvaTestCase): def test_bin_ops(self): """ Test all binary ops """ @@ -14,122 +15,145 @@ def test_bin_ops(self): for binOp in [lambda a, b: a + b, lambda a, b: a - b, lambda a, b: a * b]: for enc1 in [False, True]: for enc2 in [False, True]: - prog = EvaProgram('BinOp', vec_size = 64) + prog = EvaProgram("BinOp", vec_size=64) with prog: - a = Input('a', enc1) - b = Input('b', enc2) - Output('y', binOp(a,b)) + a = Input("a", enc1) + b = Input("b", enc2) + Output("y", binOp(a, b)) prog.set_output_ranges(20) prog.set_input_scales(30) - self.assert_compiles_and_matches_reference(prog, - config={'warn_vec_size':'false'}) - + self.assert_compiles_and_matches_reference( + prog, config={"warn_vec_size": "false"} + ) + def test_unary_ops(self): """ Test all unary ops """ - for unOp in [lambda x: x, lambda x: -x, lambda x: x**3, lambda x: 42]: + for unOp in [lambda x: x, lambda x: -x, lambda x: x ** 3, lambda x: 42]: for enc in [False, True]: - prog = EvaProgram('UnOp', vec_size = 64) + prog = EvaProgram("UnOp", vec_size=64) with prog: - x = Input('x', enc) - Output('y', unOp(x)) + x = Input("x", enc) + Output("y", unOp(x)) prog.set_output_ranges(20) prog.set_input_scales(30) - self.assert_compiles_and_matches_reference(prog, - config={'warn_vec_size':'false'}) + self.assert_compiles_and_matches_reference( + prog, config={"warn_vec_size": "false"} + ) def test_rotations(self): """ Test all rotations """ for rotOp in [lambda x, r: x << r, lambda x, r: x >> r]: for enc in [False, True]: - for rot in range(-2,2): - prog = EvaProgram('RotOp', vec_size = 8) + for rot in range(-2, 2): + prog = EvaProgram("RotOp", vec_size=8) with prog: - x = Input('x') - Output('y', rotOp(x,rot)) + x = Input("x") + Output("y", rotOp(x, rot)) prog.set_output_ranges(20) prog.set_input_scales(30) - self.assert_compiles_and_matches_reference(prog, - config={'warn_vec_size':'false'}) + self.assert_compiles_and_matches_reference( + prog, config={"warn_vec_size": "false"} + ) def test_unencrypted_computation(self): """ Test computation on unencrypted values """ for enc1 in [False, True]: for enc2 in [False, True]: - prog = EvaProgram('UnencryptedInputs', vec_size=128) + prog = EvaProgram("UnencryptedInputs", vec_size=128) with prog: - x1 = Input('x1', enc1) - x2 = Input('x2', enc2) - Output('y', pow(x2,3) + x1*x2) + x1 = Input("x1", enc1) + x2 = Input("x2", enc2) + Output("y", pow(x2, 3) + x1 * x2) prog.set_output_ranges(20) prog.set_input_scales(30) - self.assert_compiles_and_matches_reference(prog, - config={'warn_vec_size':'false'}) + self.assert_compiles_and_matches_reference( + prog, config={"warn_vec_size": "false"} + ) def test_security_levels(self): """ Check that all supported security levels work """ - security_levels = ['128','192','256'] - quantum_safety = ['false','true'] + security_levels = ["128", "192", "256"] + quantum_safety = ["false", "true"] for s in security_levels: for q in quantum_safety: - prog = EvaProgram('SecurityLevel', vec_size=512) + prog = EvaProgram("SecurityLevel", vec_size=512) with prog: - x = Input('x') - Output('y', 5*x*x + 3*x + x<<12 + 10) + x = Input("x") + Output("y", 5 * x * x + 3 * x + x << 12 + 10) prog.set_output_ranges(20) prog.set_input_scales(30) - self.assert_compiles_and_matches_reference(prog, - config={'security_level':s, 'quantum_safe':q, 'warn_vec_size':'false'}) - + self.assert_compiles_and_matches_reference( + prog, + config={ + "security_level": s, + "quantum_safe": q, + "warn_vec_size": "false", + }, + ) + @unittest.expectedFailure def test_unsupported_security_level(self): """ Check that unsupported security levels error out """ - prog = EvaProgram('SecurityLevel', vec_size=512) + prog = EvaProgram("SecurityLevel", vec_size=512) with prog: - x = Input('x') - Output('y', 5*x*x + 3*x + x<<12 + 10) + x = Input("x") + Output("y", 5 * x * x + 3 * x + x << 12 + 10) prog.set_output_ranges(20) prog.set_input_scales(30) - self.assert_compiles_and_matches_reference(prog, - config={'security_level':'1024','warn_vec_size':'false'}) + self.assert_compiles_and_matches_reference( + prog, config={"security_level": "1024", "warn_vec_size": "false"} + ) def test_reduction_balancer(self): """ Check that reductions are balanced under balance_reductions=true """ - prog = EvaProgram('ReductionTree', vec_size=16384) + prog = EvaProgram("ReductionTree", vec_size=16384) with prog: - x1 = Input('x1') - x2 = Input('x2') - x3 = Input('x3') - x4 = Input('x4') - Output('y', (x1*(x2*(x3*x4))) + (x1+(x2+(x3+x4)))) + x1 = Input("x1") + x2 = Input("x2") + x3 = Input("x3") + x4 = Input("x4") + Output("y", (x1 * (x2 * (x3 * x4))) + (x1 + (x2 + (x3 + x4)))) prog.set_output_ranges(20) prog.set_input_scales(60) - - progc, params, signature = self.assert_compiles_and_matches_reference(prog, - config={'rescaler':'always', 'balance_reductions':'false', 'warn_vec_size':'false'}) + + progc, params, signature = self.assert_compiles_and_matches_reference( + prog, + config={ + "rescaler": "always", + "balance_reductions": "false", + "warn_vec_size": "false", + }, + ) self.assertEqual(params.prime_bits, [60, 20, 60, 60, 60, 60]) - - progc, params, signature = self.assert_compiles_and_matches_reference(prog, - config={'rescaler':'always', 'balance_reductions':'true', 'warn_vec_size':'false'}) + + progc, params, signature = self.assert_compiles_and_matches_reference( + prog, + config={ + "rescaler": "always", + "balance_reductions": "true", + "warn_vec_size": "false", + }, + ) self.assertEqual(params.prime_bits, [60, 20, 60, 60, 60]) def test_seal_no_throw_on_transparent(self): @@ -140,75 +164,74 @@ def test_seal_no_throw_on_transparent(self): result in them. For example, x1-x2 is transparent only if the user gives the same ciphertext as both inputs.""" - prog = EvaProgram('Transparent', vec_size=4096) + prog = EvaProgram("Transparent", vec_size=4096) with prog: - x = Input('x') - Output('y', x-x+x*0) + x = Input("x") + Output("y", x - x + x * 0) prog.set_output_ranges(20) prog.set_input_scales(30) - self.assert_compiles_and_matches_reference(prog, - config={'warn_vec_size':'false'}) - + self.assert_compiles_and_matches_reference( + prog, config={"warn_vec_size": "false"} + ) + def test_serialization(self): """ Test (de)serialization and check that results stay the same """ - poly = EvaProgram('Polynomial', vec_size=4096) + poly = EvaProgram("Polynomial", vec_size=4096) with poly: - x = Input('x') - Output('y', 3*x**2 + 5*x - 2) + x = Input("x") + Output("y", 3 * x ** 2 + 5 * x - 2) poly.set_output_ranges(20) poly.set_input_scales(30) - inputs = { - 'x': [i for i in range(poly.vec_size)] - } + inputs = {"x": [i for i in range(poly.vec_size)]} reference = evaluate(poly, inputs) - compiler = CKKSCompiler(config={'warn_vec_size':'false'}) + compiler = CKKSCompiler(config={"warn_vec_size": "false"}) poly, params, signature = compiler.compile(poly) with tempfile.TemporaryDirectory() as tmp_dir: tmp_path = lambda x: os.path.join(tmp_dir, x) - save(poly, tmp_path('poly.eva')) - save(params, tmp_path('poly.evaparams')) - save(signature, tmp_path('poly.evasignature')) + save(poly, tmp_path("poly.eva")) + save(params, tmp_path("poly.evaparams")) + save(signature, tmp_path("poly.evasignature")) # Key generation time - params = load(tmp_path('poly.evaparams')) + params = load(tmp_path("poly.evaparams")) public_ctx, secret_ctx = generate_keys(params) - save(public_ctx, tmp_path('poly.sealpublic')) - save(secret_ctx, tmp_path('poly.sealsecret')) + save(public_ctx, tmp_path("poly.sealpublic")) + save(secret_ctx, tmp_path("poly.sealsecret")) # Runtime on client - signature = load(tmp_path('poly.evasignature')) - public_ctx = load(tmp_path('poly.sealpublic')) + signature = load(tmp_path("poly.evasignature")) + public_ctx = load(tmp_path("poly.sealpublic")) encInputs = public_ctx.encrypt(inputs, signature) - save(encInputs, tmp_path('poly_inputs.sealvals')) + save(encInputs, tmp_path("poly_inputs.sealvals")) # Runtime on server - poly = load(tmp_path('poly.eva')) - public_ctx = load(tmp_path('poly.sealpublic')) - encInputs = load(tmp_path('poly_inputs.sealvals')) + poly = load(tmp_path("poly.eva")) + public_ctx = load(tmp_path("poly.sealpublic")) + encInputs = load(tmp_path("poly_inputs.sealvals")) encOutputs = public_ctx.execute(poly, encInputs) - save(encOutputs, tmp_path('poly_outputs.sealvals')) + save(encOutputs, tmp_path("poly_outputs.sealvals")) # Runtime back on client - secret_ctx = load(tmp_path('poly.sealsecret')) - encOutputs = load(tmp_path('poly_outputs.sealvals')) + secret_ctx = load(tmp_path("poly.sealsecret")) + encOutputs = load(tmp_path("poly_outputs.sealvals")) outputs = secret_ctx.decrypt(encOutputs, signature) @@ -216,5 +239,6 @@ def test_serialization(self): self.assertTrue(valuation_mse(reference, reference_compiled) < 0.0000000001) self.assertTrue(valuation_mse(outputs, reference) < 0.01) -if __name__ == '__main__': - unittest.main() \ No newline at end of file + +if __name__ == "__main__": + unittest.main() diff --git a/tests/large_programs.py b/tests/large_programs.py index ee63b8c..a6a25a8 100644 --- a/tests/large_programs.py +++ b/tests/large_programs.py @@ -6,6 +6,7 @@ from common import * from eva import EvaProgram, Input, Output + class LargePrograms(EvaTestCase): def test_sobel_configs(self): """ Check accuracy of Sobel filter on random image with various compiler configurations """ @@ -27,40 +28,42 @@ def convolutionXY(image, width, filter): h = 90 w = 90 - sobel = EvaProgram('sobel', vec_size=2**(math.ceil(math.log(h*w, 2)))) + sobel = EvaProgram("sobel", vec_size=2 ** (math.ceil(math.log(h * w, 2)))) with sobel: - image = Input('image') + image = Input("image") - sobel_filter = [ - [-1, 0, 1], - [-2, 0, 2], - [-1, 0, 1]] + sobel_filter = [[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]] a1 = 2.2137874823876622 a2 = -1.0984324107372518 a3 = 0.17254603006834726 conv_hor, conv_ver = convolutionXY(image, w, sobel_filter) - x = conv_hor**2 + conv_ver**2 - Output('image', x * a1 + x**2 * a2 + x**3 * a3) + x = conv_hor ** 2 + conv_ver ** 2 + Output("image", x * a1 + x ** 2 * a2 + x ** 3 * a3) sobel.set_input_scales(45) sobel.set_output_ranges(20) - for rescaler in ['lazy_waterline','eager_waterline','always']: - for balance_reductions in ['true','false']: - self.assert_compiles_and_matches_reference(sobel, - config={'rescaler':rescaler,'balance_reductions':balance_reductions}) + for rescaler in ["lazy_waterline", "eager_waterline", "always"]: + for balance_reductions in ["true", "false"]: + self.assert_compiles_and_matches_reference( + sobel, + config={ + "rescaler": rescaler, + "balance_reductions": balance_reductions, + }, + ) def test_regression(self): """ Test batched compilation and execution of multiple linear regression programs """ - - linreg = EvaProgram('linear_regression', vec_size=2048) + + linreg = EvaProgram("linear_regression", vec_size=2048) with linreg: p = 63 - x = [Input(f'x{i}') for i in range(p)] - e = Input('e') + x = [Input(f"x{i}") for i in range(p)] + e = Input("e") b0 = 6.56 b = [i * 0.732 for i in range(p)] @@ -69,21 +72,23 @@ def test_regression(self): t = x[i] * b[i] y += t - Output('y', y) + Output("y", y) linreg.set_input_scales(40) linreg.set_output_ranges(30) - linreg_inputs = {'e': [(linreg.vec_size - i) * 0.001 for i in range(linreg.vec_size)]} + linreg_inputs = { + "e": [(linreg.vec_size - i) * 0.001 for i in range(linreg.vec_size)] + } for i in range(p): - linreg_inputs[f'x{i}'] = [i * j * 0.01 for j in range(linreg.vec_size)] + linreg_inputs[f"x{i}"] = [i * j * 0.01 for j in range(linreg.vec_size)] - polyreg = EvaProgram('polynomial_regression', vec_size=4096) + polyreg = EvaProgram("polynomial_regression", vec_size=4096) with polyreg: p = 4 - x = Input('x') - e = Input('e') + x = Input("x") + e = Input("e") b0 = 6.56 b = [i * 0.732 for i in range(p)] @@ -95,23 +100,23 @@ def test_regression(self): t = x_i * b[i] y += t - Output('y', y) + Output("y", y) polyreg.set_input_scales(40) polyreg.set_output_ranges(30) polyreg_inputs = { - 'x': [i * 0.01 for i in range(polyreg.vec_size)], - 'e': [(polyreg.vec_size - i) * 0.001 for i in range(polyreg.vec_size)], + "x": [i * 0.01 for i in range(polyreg.vec_size)], + "e": [(polyreg.vec_size - i) * 0.001 for i in range(polyreg.vec_size)], } - multireg = EvaProgram('multivariate_regression', vec_size=2048) + multireg = EvaProgram("multivariate_regression", vec_size=2048) with multireg: p = 63 k = 4 - x = [Input(f'x{i}') for i in range(p)] - e = [Input(f'e{j}') for j in range(k)] + x = [Input(f"x{i}") for i in range(p)] + e = [Input(f"e{j}") for j in range(k)] b0 = [j * 0.56 for j in range(k)] b = [[k * i * 0.732 for i in range(p)] for j in range(k)] @@ -123,20 +128,26 @@ def test_regression(self): y[j] += t for j in range(k): - Output(f'y{j}', y[j]) + Output(f"y{j}", y[j]) multireg.set_input_scales(40) multireg.set_output_ranges(30) multireg_inputs = {} for i in range(p): - multireg_inputs[f'x{i}'] = [i * j * 0.01 for j in range(multireg.vec_size)] + multireg_inputs[f"x{i}"] = [i * j * 0.01 for j in range(multireg.vec_size)] for j in range(k): - multireg_inputs[f'e{j}'] = [(multireg.vec_size - i) * j * 0.001 for i in range(multireg.vec_size)] - - compiler = CKKSCompiler(config={'warn_vec_size':'false'}) + multireg_inputs[f"e{j}"] = [ + (multireg.vec_size - i) * j * 0.001 for i in range(multireg.vec_size) + ] - for prog, inputs in [(linreg, linreg_inputs), (polyreg, polyreg_inputs), (multireg, multireg_inputs)]: + compiler = CKKSCompiler(config={"warn_vec_size": "false"}) + + for prog, inputs in [ + (linreg, linreg_inputs), + (polyreg, polyreg_inputs), + (multireg, multireg_inputs), + ]: compiled_prog, params, signature = compiler.compile(prog) public_ctx, secret_ctx = generate_keys(params) enc_inputs = public_ctx.encrypt(inputs, signature) @@ -145,5 +156,6 @@ def test_regression(self): reference = evaluate(compiled_prog, inputs) self.assertTrue(valuation_mse(outputs, reference) < 0.01) -if __name__ == '__main__': - unittest.main() \ No newline at end of file + +if __name__ == "__main__": + unittest.main() diff --git a/tests/std.py b/tests/std.py index f24beb6..c7b2005 100644 --- a/tests/std.py +++ b/tests/std.py @@ -6,33 +6,37 @@ from eva import EvaProgram, Input, Output from eva.std.numeric import horizontal_sum + class Std(EvaTestCase): def test_horizontal_sum(self): """ Test eva.std.numeric.horizontal_sum """ for enc in [True, False]: - prog = EvaProgram('HorizontalSum', vec_size = 2048) + prog = EvaProgram("HorizontalSum", vec_size=2048) with prog: - x = Input('x', is_encrypted=enc) + x = Input("x", is_encrypted=enc) y = horizontal_sum(x) - Output('y', y) + Output("y", y) prog.set_output_ranges(25) prog.set_input_scales(33) - self.assert_compiles_and_matches_reference(prog, - config={'warn_vec_size':'false'}) + self.assert_compiles_and_matches_reference( + prog, config={"warn_vec_size": "false"} + ) - prog = EvaProgram('HorizontalSumConstant', vec_size = 2048) + prog = EvaProgram("HorizontalSumConstant", vec_size=2048) with prog: y = horizontal_sum([1 for _ in range(prog.vec_size)]) - Output('y', y) + Output("y", y) prog.set_output_ranges(25) prog.set_input_scales(33) - self.assert_compiles_and_matches_reference(prog, - config={'warn_vec_size':'false'}) + self.assert_compiles_and_matches_reference( + prog, config={"warn_vec_size": "false"} + ) + -if __name__ == '__main__': - unittest.main() \ No newline at end of file +if __name__ == "__main__": + unittest.main() diff --git a/third_party/Galois b/third_party/Galois deleted file mode 160000 index 306535c..0000000 --- a/third_party/Galois +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 306535c4931b8d398518624b9b6428f7120a0b44 diff --git a/third_party/pybind11 b/third_party/pybind11 deleted file mode 160000 index f1abf5d..0000000 --- a/third_party/pybind11 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f1abf5d9159b805674197f6bc443592e631c9130