From 36928a70676462ab64e2e7cc9f18704eb46df008 Mon Sep 17 00:00:00 2001 From: EchoFan Date: Fri, 21 Jul 2023 18:15:44 +0900 Subject: [PATCH 1/5] add aws lambda build --- .dockerignore | 1 + .github/workflows/build_release_aws_x64.yml | 53 +++++++++++++++++++++ Dockerfile.buildaws | 29 +++++++++++ Dockerfile.debugaws | 12 +++++ examples/index.js | 2 +- 5 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build_release_aws_x64.yml create mode 100644 Dockerfile.buildaws create mode 100644 Dockerfile.debugaws diff --git a/.dockerignore b/.dockerignore index 393447e..3699df2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ +.git build/ node_modules/ docs/ diff --git a/.github/workflows/build_release_aws_x64.yml b/.github/workflows/build_release_aws_x64.yml new file mode 100644 index 0000000..a031569 --- /dev/null +++ b/.github/workflows/build_release_aws_x64.yml @@ -0,0 +1,53 @@ +name: Build Pre-build Packages for AWS + +on: + workflow_dispatch: + push: + tags: + - "*" + pull_request: + branches: [ "main" ] + +concurrency: + group: release + cancel-in-progress: false + +jobs: + build: + environment: release + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v3 + with: + submodules: "true" + - name: Setup Node.js 18 + uses: actions/setup-node@v3 + with: + node-version: 18 + - name: NPM Pack + run: npm pack + - name: Rename Package + run: mv faiss-node-*.tgz faiss-node.tgz + - name: Build & test + run: docker build -f Dockerfile.buildaws -t faiss-build-aws . + - name: Build & test + run: | + tempid=$(docker create faiss-build-aws) + docker cp $tempid:/var/task aws + docker rm -v $tempid + cd aws && ls -laht && zip -r aws.zip . + - name: Get Version + id: version + run: | + VERSION=$(node -p -e "require('./package.json').version") + echo $VERSION + echo VERSION=$VERSION >> "$GITHUB_OUTPUT" + # - name: Upload to Release + # uses: softprops/action-gh-release@v1 + # with: + # draft: true + # fail_on_unmatched_files: true + # files: aws/*.zip + # tag_name: "v${{ steps.version.outputs.VERSION }}" diff --git a/Dockerfile.buildaws b/Dockerfile.buildaws new file mode 100644 index 0000000..14102e4 --- /dev/null +++ b/Dockerfile.buildaws @@ -0,0 +1,29 @@ +FROM public.ecr.aws/lambda/nodejs:18 as build +COPY package*.json ./ +RUN npm ci --ignore-scripts + +RUN yum -y install wget tar xz gzip gcc gcc-c++ make lapack-devel patchelf +RUN wget -q https://github.com/Kitware/CMake/releases/download/v3.27.0/cmake-3.27.0-linux-x86_64.sh && \ + sh cmake-3.27.0-linux-x86_64.sh --skip-license --exclude-subdir --prefix=/usr/local + +RUN patchelf --set-rpath "\$ORIGIN/" /lib64/libgfortran.so.4 + +ENTRYPOINT ["tail"] +CMD ["-f","/dev/null"] +COPY . . +RUN npm run build:test + + +FROM public.ecr.aws/lambda/nodejs:18 +COPY faiss-node.tgz ./ +RUN npm init -y && npm i faiss-node.tgz --ignore-scripts && rm faiss-node.tgz +COPY examples/index.js index.js +COPY --from=build /var/task/build/Release/faiss-node.node node_modules/faiss-node/build/Release/faiss-node.node +COPY --from=build /var/task/build/Release/libfaiss.a node_modules/faiss-node/build/Release/libfaiss.a +COPY --from=build /lib64/libgomp.so.1 node_modules/faiss-node/build/Release/libgomp.so.1 +COPY --from=build /lib64/libpthread.so.0 node_modules/faiss-node/build/Release/libpthread.so.0 +COPY --from=build /lib64/libblas.so.3 node_modules/faiss-node/build/Release/libblas.so.3 +COPY --from=build /lib64/liblapack.so.3 node_modules/faiss-node/build/Release/liblapack.so.3 +COPY --from=build /lib64/libgfortran.so.4 node_modules/faiss-node/build/Release/libgfortran.so.4 +COPY --from=build /lib64/libquadmath.so.0 node_modules/faiss-node/build/Release/libquadmath.so.0 +RUN node index.js \ No newline at end of file diff --git a/Dockerfile.debugaws b/Dockerfile.debugaws new file mode 100644 index 0000000..d1bbb30 --- /dev/null +++ b/Dockerfile.debugaws @@ -0,0 +1,12 @@ +FROM public.ecr.aws/lambda/nodejs:18 +COPY package*.json ./ +RUN npm ci --ignore-scripts + +RUN yum -y install wget tar xz gzip gcc gcc-c++ make lapack-devel +RUN wget -q https://github.com/Kitware/CMake/releases/download/v3.27.0/cmake-3.27.0-linux-x86_64.sh && \ + sh cmake-3.27.0-linux-x86_64.sh --skip-license --exclude-subdir --prefix=/usr/local + +COPY . . + +ENTRYPOINT ["tail"] +CMD ["-f","/dev/null"] \ No newline at end of file diff --git a/examples/index.js b/examples/index.js index 86c796f..1c85f3e 100644 --- a/examples/index.js +++ b/examples/index.js @@ -1,4 +1,4 @@ -const { IndexFlatL2 } = require('../lib/index'); +const { IndexFlatL2 } = require('faiss-node'); const dimension = 2; const index = new IndexFlatL2(dimension); From 98473e3acc5a2076fa3d0caa40f7f7ade3d19366 Mon Sep 17 00:00:00 2001 From: EchoFan Date: Fri, 21 Jul 2023 18:39:12 +0900 Subject: [PATCH 2/5] add mjs for test --- .github/workflows/build_release_aws_x64.yml | 12 ++++++------ Dockerfile.buildaws | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_release_aws_x64.yml b/.github/workflows/build_release_aws_x64.yml index a031569..edd9cdc 100644 --- a/.github/workflows/build_release_aws_x64.yml +++ b/.github/workflows/build_release_aws_x64.yml @@ -32,18 +32,18 @@ jobs: run: mv faiss-node-*.tgz faiss-node.tgz - name: Build & test run: docker build -f Dockerfile.buildaws -t faiss-build-aws . - - name: Build & test - run: | - tempid=$(docker create faiss-build-aws) - docker cp $tempid:/var/task aws - docker rm -v $tempid - cd aws && ls -laht && zip -r aws.zip . - name: Get Version id: version run: | VERSION=$(node -p -e "require('./package.json').version") echo $VERSION echo VERSION=$VERSION >> "$GITHUB_OUTPUT" + - name: Package + run: | + tempid=$(docker create faiss-build-aws) + docker cp $tempid:/var/task aws + docker rm -v $tempid + cd aws && ls -laht && zip -r faiss-node-v${{ steps.version.outputs.VERSION }}-aws-x64.zip . # - name: Upload to Release # uses: softprops/action-gh-release@v1 # with: diff --git a/Dockerfile.buildaws b/Dockerfile.buildaws index 14102e4..e529bfc 100644 --- a/Dockerfile.buildaws +++ b/Dockerfile.buildaws @@ -17,6 +17,7 @@ RUN npm run build:test FROM public.ecr.aws/lambda/nodejs:18 COPY faiss-node.tgz ./ RUN npm init -y && npm i faiss-node.tgz --ignore-scripts && rm faiss-node.tgz +COPY examples/index-aws.mjs index.mjs COPY examples/index.js index.js COPY --from=build /var/task/build/Release/faiss-node.node node_modules/faiss-node/build/Release/faiss-node.node COPY --from=build /var/task/build/Release/libfaiss.a node_modules/faiss-node/build/Release/libfaiss.a From abe736c1d9d41cf12a4ab5b7711642968f2c543f Mon Sep 17 00:00:00 2001 From: EchoFan Date: Fri, 21 Jul 2023 18:39:39 +0900 Subject: [PATCH 3/5] add missing file --- examples/index-aws.mjs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 examples/index-aws.mjs diff --git a/examples/index-aws.mjs b/examples/index-aws.mjs new file mode 100644 index 0000000..2c6df51 --- /dev/null +++ b/examples/index-aws.mjs @@ -0,0 +1,24 @@ +import pkg from 'faiss-node' + +export const handler = async(event) => { + const { IndexFlatL2 } = pkg + const dimension = 2; + const index = new IndexFlatL2(dimension); + + console.log(index.getDimension()); + console.log(index.isTrained()); + console.log(index.ntotal()); + index.add([1, 0]); + index.add([1, 2]); + index.add([1, 3]); + index.add([1, 1]); + console.log(index.ntotal()); + + const k = 4; + const results = index.search([1, 0], k); + console.log(results.labels); + console.log(results.distances); + + return results; +}; + From 735ea68cabfaf439853146211c904424fabd2ef8 Mon Sep 17 00:00:00 2001 From: EchoFan Date: Sun, 23 Jul 2023 23:56:41 +0900 Subject: [PATCH 4/5] creat layer and remove useless code --- .github/workflows/build_release_aws_x64.yml | 4 +++- Dockerfile.buildaws | 5 ++--- Dockerfile.debugaws | 2 +- examples/{index-aws.mjs => index-aws.js} | 6 ++---- 4 files changed, 8 insertions(+), 9 deletions(-) rename examples/{index-aws.mjs => index-aws.js} (81%) diff --git a/.github/workflows/build_release_aws_x64.yml b/.github/workflows/build_release_aws_x64.yml index edd9cdc..32b6706 100644 --- a/.github/workflows/build_release_aws_x64.yml +++ b/.github/workflows/build_release_aws_x64.yml @@ -43,7 +43,9 @@ jobs: tempid=$(docker create faiss-build-aws) docker cp $tempid:/var/task aws docker rm -v $tempid - cd aws && ls -laht && zip -r faiss-node-v${{ steps.version.outputs.VERSION }}-aws-x64.zip . + cd aws && ls -laht + mkdir nodejs && mv node_modules nodejs/node_modules && ln -s nodejs/node_modules node_modules + zip -r --symlinks faiss-node-v${{ steps.version.outputs.VERSION }}-aws-x64.zip . # - name: Upload to Release # uses: softprops/action-gh-release@v1 # with: diff --git a/Dockerfile.buildaws b/Dockerfile.buildaws index e529bfc..10b6444 100644 --- a/Dockerfile.buildaws +++ b/Dockerfile.buildaws @@ -2,7 +2,7 @@ FROM public.ecr.aws/lambda/nodejs:18 as build COPY package*.json ./ RUN npm ci --ignore-scripts -RUN yum -y install wget tar xz gzip gcc gcc-c++ make lapack-devel patchelf +RUN yum -y install wget tar gzip gcc gcc-c++ make lapack-devel patchelf RUN wget -q https://github.com/Kitware/CMake/releases/download/v3.27.0/cmake-3.27.0-linux-x86_64.sh && \ sh cmake-3.27.0-linux-x86_64.sh --skip-license --exclude-subdir --prefix=/usr/local @@ -17,8 +17,7 @@ RUN npm run build:test FROM public.ecr.aws/lambda/nodejs:18 COPY faiss-node.tgz ./ RUN npm init -y && npm i faiss-node.tgz --ignore-scripts && rm faiss-node.tgz -COPY examples/index-aws.mjs index.mjs -COPY examples/index.js index.js +COPY examples/index-aws.js index.js COPY --from=build /var/task/build/Release/faiss-node.node node_modules/faiss-node/build/Release/faiss-node.node COPY --from=build /var/task/build/Release/libfaiss.a node_modules/faiss-node/build/Release/libfaiss.a COPY --from=build /lib64/libgomp.so.1 node_modules/faiss-node/build/Release/libgomp.so.1 diff --git a/Dockerfile.debugaws b/Dockerfile.debugaws index d1bbb30..7a4cab4 100644 --- a/Dockerfile.debugaws +++ b/Dockerfile.debugaws @@ -2,7 +2,7 @@ FROM public.ecr.aws/lambda/nodejs:18 COPY package*.json ./ RUN npm ci --ignore-scripts -RUN yum -y install wget tar xz gzip gcc gcc-c++ make lapack-devel +RUN yum -y install wget tar gzip gcc gcc-c++ make lapack-devel RUN wget -q https://github.com/Kitware/CMake/releases/download/v3.27.0/cmake-3.27.0-linux-x86_64.sh && \ sh cmake-3.27.0-linux-x86_64.sh --skip-license --exclude-subdir --prefix=/usr/local diff --git a/examples/index-aws.mjs b/examples/index-aws.js similarity index 81% rename from examples/index-aws.mjs rename to examples/index-aws.js index 2c6df51..76f432d 100644 --- a/examples/index-aws.mjs +++ b/examples/index-aws.js @@ -1,7 +1,6 @@ -import pkg from 'faiss-node' +const { IndexFlatL2 } = require('faiss-node'); -export const handler = async(event) => { - const { IndexFlatL2 } = pkg +exports.handler = async function(event, context) { const dimension = 2; const index = new IndexFlatL2(dimension); @@ -21,4 +20,3 @@ export const handler = async(event) => { return results; }; - From e8ea8b6a5aa476a2d59cd995f9236d25d68e714b Mon Sep 17 00:00:00 2001 From: EchoFan Date: Mon, 24 Jul 2023 10:20:55 +0900 Subject: [PATCH 5/5] [skip ci] enable upload and remove pr trigger --- .github/workflows/build_release_aws_x64.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_release_aws_x64.yml b/.github/workflows/build_release_aws_x64.yml index 32b6706..14848ae 100644 --- a/.github/workflows/build_release_aws_x64.yml +++ b/.github/workflows/build_release_aws_x64.yml @@ -5,8 +5,6 @@ on: push: tags: - "*" - pull_request: - branches: [ "main" ] concurrency: group: release @@ -46,10 +44,10 @@ jobs: cd aws && ls -laht mkdir nodejs && mv node_modules nodejs/node_modules && ln -s nodejs/node_modules node_modules zip -r --symlinks faiss-node-v${{ steps.version.outputs.VERSION }}-aws-x64.zip . - # - name: Upload to Release - # uses: softprops/action-gh-release@v1 - # with: - # draft: true - # fail_on_unmatched_files: true - # files: aws/*.zip - # tag_name: "v${{ steps.version.outputs.VERSION }}" + - name: Upload to Release + uses: softprops/action-gh-release@v1 + with: + draft: true + fail_on_unmatched_files: true + files: aws/*.zip + tag_name: "v${{ steps.version.outputs.VERSION }}"