DATAUP-735 Develop->Main

.dockerignore

@@ -0,0 +1,51 @@
+.dockerignore
+.travis.yml
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.Python
+.tox
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+*.log
+.git
+.git*
+CODEOWNERS
+LICENSE.md
+Makefile
+hooks
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+/dist/
+/client_src/dist/
+eggs/
+.eggs/
+*.egg-info/
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# docker bits
+Dockerfile*
+docker-compose*

.gitattributes

@@ -0,0 +1,7 @@
+/CODEOWNERS export-ignore
+/.gitattributes export-ignore
+/.gitignore export-ignore
+/.travis.yml export-ignore
+README.md export-ignore
+/Makefile export-ignore
+/test export-ignore

.github/CONTRIBUTING.md

@@ -0,0 +1,22 @@
+# Contributing to KBase
+
+Important resources:
+
+* [The KBase Code of Conduct](https://kbase.us/code-of-conduct/)
+* [The KBase user documentation](https://kbase.us/new-to-kbase/)
+* [The KBase users' slack channel](https://kbaseusers.slack.com)
+* [The KBase SDK docs](https://kbase.github.io/kb_sdk_docs/)
+* [The kbase Github organization](https://github.com/kbase)
+* [The kbaseapps Github organization](https://github.com/kbaseapps)
+
+## Submitting changes
+
+After opening a Github pull request on the repo you'd like to update, be sure to:
+
+* Update documentation, including the README.md file, any additional documentation under `/docs`, and any separate documentation sites that cover the functionality.
+* Check that you added test coverage for any changes you made.
+* Check if your changes break an API. If so, increment the version and ensure that any current dependents will not break.
+
+Always write a clear log messages for your commits.
+
+:tada: :100: Thank you for your contributions!

.github/pull_request_template.md

@@ -0,0 +1,8 @@
+
+- [ ] I updated the README.md docs to reflect this change.
+
+For changes to the codebase:
+
+- [ ] I have written tests to cover this change.
+- [ ] This is not a breaking API change OR
+- [ ] This is a breaking API change and I have incremented the API version and added a summary to CHANGELOG.md.

.github/workflows/build_prodrc_pr.yaml

@@ -0,0 +1,31 @@
+---
+name: Build Prod RC Image
+'on':
+  pull_request:
+    branches:
+    - master
+    - main
+    types:
+    - opened
+    - synchronize
+    - ready_for_review
+jobs:
+  docker_build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Verify merge is develop -> main
+      if: github.head_ref != 'develop'
+      run: echo "Must merge from develop -> main/master"; exit 1
+    - name: Check out GitHub Repo
+      if: github.event.pull_request.draft == false && github.head_ref == 'develop'
+      with:
+        ref: "${{ github.event.pull_request.head.sha }}"
+      uses: actions/checkout@v2
+    - name: Build and Push to Packages
+      if: github.event.pull_request.draft == false && github.head_ref == 'develop'
+      env:
+        PR: "${{ github.event.pull_request.number }}"
+        SHA: "${{ github.event.pull_request.head.sha }}"
+        DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}"
+        DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}"
+      run: "./.github/workflows/scripts/build_prodrc_pr.sh\n"

.github/workflows/build_test_pr.yaml

@@ -0,0 +1,27 @@
+---
+name: Build Test Image
+'on':
+  pull_request:
+    branches:
+    - develop
+    types:
+    - opened
+    - synchronize
+    - ready_for_review
+jobs:
+  docker_build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out GitHub Repo
+      if: github.event.pull_request.draft == false
+      with:
+        ref: "${{ github.event.pull_request.head.sha }}"
+      uses: actions/checkout@v2
+    - name: Build and Push to Packages
+      if: github.event.pull_request.draft == false
+      env:
+        PR: "${{ github.event.pull_request.number }}"
+        SHA: "${{ github.event.pull_request.head.sha }}"
+        DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}"
+        DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}"
+      run: "./.github/workflows/scripts/build_test_pr.sh\n"

.github/workflows/prod_release.yaml

@@ -0,0 +1,38 @@
+---
+name: Publish Release Image
+'on':
+  release:
+    branches:
+    - main
+    - master
+    types:
+    - published
+jobs:
+  docker_build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check Tag
+      id: check-tag
+      run: |-
+        if [[ ${{ github.ref_name }} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+          echo ::set-output name=match::true
+        fi
+    - name: Report SemVer Check
+      if: steps.check-tag.outputs.match != 'true'
+      run: echo "Release version must follow semantic naming (e.g. 1.0.2)"; exit 1
+    - name: Check Source Branch
+      if: github.event.release.target_commitish != 'master' && github.event.release.target_commitish != 'main'
+      run: echo "Releases must be built from master/main branch"; exit 1
+    - name: Check out GitHub Repo
+      with:
+        ref: "${{ github.event.pull_request.head.sha }}"
+      uses: actions/checkout@v2
+    - name: Build and Push to Packages
+      env:
+        ISH: "${{ github.event.release.target_commitish }}"
+        PR: "${{ github.event.pull_request.number }}"
+        SHA: "${{ github.event.pull_request.head.sha }}"
+        VER: "${{ github.event.release.tag_name }}"
+        DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}"
+        DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}"
+      run: "./.github/workflows/scripts/prod_release.sh\n"

.github/workflows/run_tests.yaml

@@ -0,0 +1,49 @@
+name: Relation Engine test and deploy
+on:
+  [push, pull_request]
+jobs:
+  run_tests:
+    runs-on: ubuntu-latest
+    steps:
+    - name: checkout git repo
+      uses: actions/checkout@v2
+
+    - name: run tests
+      shell: bash
+      env:
+        GITHUB_ACTIONS_CI: 1
+      run: |
+        docker-compose build
+        docker-compose run re_api sh scripts/run_tests.sh
+        docker-compose down --remove-orphans
+
+  docker_build_and_push:
+    runs-on: ubuntu-latest
+    needs: run_tests
+    if: (github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master') && github.event_name == 'push' && !contains(github.event.head_commit.message, 'skip_docker_build')
+    steps:
+    - name: checkout git repo
+      uses: actions/checkout@v2
+
+    - name: copy VERSION to TAG_NAME
+      shell: bash
+      run: |
+        mkdir -p .target
+        cp VERSION .target/TAG_NAME
+
+    - name: set env vars
+      shell: bash
+      run: |
+        echo "DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
+        echo "BRANCH=$(git symbolic-ref --short HEAD)" >> $GITHUB_ENV
+        echo "COMMIT=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+    - name: build and push to dockerhub
+      uses: opspresso/action-docker@master
+      with:
+        args: --docker
+      env:
+        USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        DOCKERFILE: "Dockerfile"
+        IMAGE_NAME: "kbase/relation_engine_api"

.github/workflows/scripts/build_prodrc_pr.sh

@@ -0,0 +1,17 @@
+#! /usr/bin/env bash
+
+export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}')
+export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')
+export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export COMMIT=$(echo "$SHA" | cut -c -7)
+
+echo "Branch is:" ${GITHUB_HEAD_REF}
+docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io
+docker build --build-arg BUILD_DATE="$DATE" \
+             --build-arg COMMIT="$COMMIT" \
+             --build-arg BRANCH="$GITHUB_HEAD_REF" \
+             --build-arg PULL_REQUEST="$PR" \
+             --label us.kbase.vcs-pull-req="$PR" \
+             -t ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" .
+docker push ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR"

.github/workflows/scripts/build_test_pr.sh

@@ -0,0 +1,17 @@
+#! /usr/bin/env bash
+
+export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}')
+export MY_APP=$(echo $(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')"-develop")
+export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export COMMIT=$(echo "$SHA" | cut -c -7)
+
+echo $DOCKER_TOKEN | docker login ghcr.io -u $DOCKER_ACTOR --password-stdin
+docker build --build-arg BUILD_DATE="$DATE" \
+             --build-arg COMMIT="$COMMIT" \
+             --build-arg BRANCH="$GITHUB_HEAD_REF" \
+             --build-arg PULL_REQUEST="$PR" \
+             --label us.kbase.vcs-pull-req="$PR" \
+             -t ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" .
+docker push ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR"
+

.github/workflows/scripts/deploy_tag.sh

@@ -0,0 +1,34 @@
+#! /usr/bin/env bash
+
+# Usage: ./deploy_tag.sh -e TARGET -o ORG -r REPO -s DEV_PROD -t IMAGE_TAG
+#
+# Example 1: ./deploy_tag.sh -o "kbase" -r "narrative-traefiker" -s "dev" -t "pr-9001" -e "ci"
+# Example 2: ./deploy_tag.sh -o "kbase" -r "narrative" -s "prod" -t "latest" -e "next"
+#
+# Where:
+#   -o ORG is the organization (`kbase`, `kbaseapps`, etc.)
+#   -r REPO is the repository (e.g. `narrative`)
+#   -s DEV_PROD determines whether to pull the development {APPNAME}-develop or production {APPNAME} image.
+#   -t IMAGE_TAG is the *current* Docker image tag, typically `pr-#` or `latest`
+#   -e TARGET is one of: `appdsshev`, `ci`, or `next`
+#
+# Be sure to set $TOKEN first!
+# See: https://docs.github.com/en/packages/getting-started-with-github-container-registry/migrating-to-github-container-registry-for-docker-images#authenticating-with-the-container-registry
+
+
+while getopts e:o:r:s:t: option
+  do
+   case "${option}"
+    in
+      e) TARGET=${OPTARG};;
+      o) ORG=${OPTARG};;
+      r) REPO=${OPTARG};;
+      s) DEV_PROD=${OPTARG};;
+      t) IMAGE_TAG=${OPTARG};;
+    esac
+done
+
+curl -H "Authorization: token $TOKEN" \
+    -H 'Accept: application/vnd.github.everest-preview+json' \
+    "https://api.github.com/repos/$ORG/$REPO/dispatches" \
+    -d '{"event_type":"Tag '"$DEV_PROD"' '"$IMAGE_TAG"' for '"$TARGET"'", "client_payload": {"image_tag": "'"$IMAGE_TAG"'","target": "'"$TARGET"'","dev_prod": "'"$DEV_PROD"'"}}'

.github/workflows/scripts/prod_release.sh

@@ -0,0 +1,24 @@
+#! /usr/bin/env bash
+
+export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}')
+export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')
+export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export COMMIT=$(echo "$SHA" | cut -c -7)
+
+echo "ISH is:" $ISH
+echo "GITHUB_REF is:" $GITHUB_REF
+echo "HEAD_REF is:" $GITHUB_HEAD_REF
+echo "BASE_REF is:" $GITHUB_BASE_REF
+echo "Release is:" $GITHUB_REF_NAME
+echo $DOCKER_TOKEN | docker login ghcr.io -u $DOCKER_ACTOR --password-stdin
+docker build --build-arg BUILD_DATE="$DATE" \
+             --build-arg COMMIT="$COMMIT" \
+             --build-arg BRANCH="$GITHUB_HEAD_REF" \
+             --build-arg PULL_REQUEST="$PR" \
+             --build-arg VERSION="$VER"  \
+             --label us.kbase.vcs-pull-req="$PR" \
+             -t ghcr.io/"$MY_ORG"/"$MY_APP":"$VER" \
+             -t ghcr.io/"$MY_ORG"/"$MY_APP":"latest" .
+docker push ghcr.io/"$MY_ORG"/"$MY_APP":"$VER"
+docker push ghcr.io/"$MY_ORG"/"$MY_APP":"latest"

.github/workflows/scripts/tag_environments.sh

@@ -0,0 +1,22 @@
+
+#! /usr/bin/env bash
+# Add vars for PR & environments to yaml, as called from external script
+
+export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}')
+export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')
+export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export COMMIT=$(echo "$SHA" | cut -c -7)
+
+if [ $DEV_PROD = "dev" ] || [ $DEV_PROD = "develop" ]
+then
+  IMAGE=$MY_APP"-develop"
+else
+  IMAGE=$MY_APP
+fi
+
+echo "Dev or Prod:" $DEV_PROD
+docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io
+docker pull ghcr.io/"$MY_ORG"/"$IMAGE":"$IMAGE_TAG"
+docker tag ghcr.io/"$MY_ORG"/"$IMAGE":"$IMAGE_TAG" ghcr.io/"$MY_ORG"/"$IMAGE":"$TARGET"
+docker push ghcr.io/"$MY_ORG"/"$IMAGE":"$TARGET"

.github/workflows/scripts/tag_prod_latest.sh

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}')
+export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')
+export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export COMMIT=$(echo "$SHA" | cut -c -7)
+
+docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io
+docker pull ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR"
+docker tag ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" ghcr.io/"$MY_ORG"/"$MY_APP":"latest-rc"
+docker push ghcr.io/"$MY_ORG"/"$MY_APP":"latest-rc"

.github/workflows/scripts/tag_test_latest.sh

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}')
+export MY_APP=$(echo $(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')"-develop")
+export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export COMMIT=$(echo "$SHA" | cut -c -7)
+
+docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io
+docker pull ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR"
+docker tag ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" ghcr.io/"$MY_ORG"/"$MY_APP":"latest"
+docker push ghcr.io/"$MY_ORG"/"$MY_APP":"latest"