From 78dabddd8f200ebc19d8bd31e4d17142b203af6a Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Wed, 24 Jun 2020 23:13:37 -0700 Subject: [PATCH 1/2] generate go client from openapi spec --- .github/workflows/cancel.yml | 2 +- .github/workflows/ci.yml | 26 ++++++++++ .pre-commit-config.yaml | 2 +- chart/values.yaml | 36 +++++++------- clients/README.md | 30 +++++++++++ clients/gen/common.sh | 35 +++++++++++++ clients/gen/go.sh | 77 +++++++++++++++++++++++++++++ scripts/ci/docker-compose/local.yml | 4 +- 8 files changed, 190 insertions(+), 22 deletions(-) create mode 100644 clients/README.md create mode 100755 clients/gen/common.sh create mode 100755 clients/gen/go.sh diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml index 5a26c0f0c4795..e831a2830b4ef 100644 --- a/.github/workflows/cancel.yml +++ b/.github/workflows/cancel.yml @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +--- name: Cancel on: [push, pull_request] jobs: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3b0e351b9a8b..b6061d781bf69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -171,6 +171,32 @@ jobs: echo "::set-output name=count::$?" id: trigger-tests + test-openapi-client-generation: + name: "Test OpenAPI client generation" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + with: + fetch-depth: 0 + - name: "Check spec file change" + run: | + URL="https://raw.githubusercontent.com/apache/airflow/master/airflow/api_connexion/openapi/v1.yaml" + curl -sL -o /tmp/current_v1.yaml "${URL}" + if diff /tmp/current_v1.yaml ./airflow/api_connexion/openapi/v1.yaml; then + echo "no openapi spec change detected, going to skip client code gen validation." + else + echo "openapi spec change detected." + echo "::set-env name=API_SPEC_CHANGED::true" + fi + - name: "Generate new clients" + if: success() && env.API_SPEC_CHANGED == 'true' + run: | + mkdir -p ./clients/go/airflow + ./clients/gen/go.sh ./airflow/api_connexion/openapi/v1.yaml ./clients/go/airflow + mkdir -p ./clients/go_master/airflow + ./clients/gen/go.sh /tmp/current_v1.yaml ./clients/go_master/airflow + diff ./clients/go_master/airflow ./clients/go/airflow || true + tests-kubernetes: timeout-minutes: 80 name: "K8s: ${{matrix.kube-mode}} ${{matrix.python-version}} ${{matrix.kubernetes-version}}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bd5ce69e5a1b7..f2b1cdd9578f4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: rev: v1.1.7 hooks: - id: forbid-tabs - exclude: ^docs/Makefile$ + exclude: ^docs/Makefile$|^clients/gen/go.sh - id: insert-license name: Add license for all SQL files files: \.sql$ diff --git a/chart/values.yaml b/chart/values.yaml index dc13064927bfd..a469dfc586e12 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -223,12 +223,12 @@ webserver: extraNetworkPolicies: [] resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi # Create initial user. defaultUser: @@ -265,12 +265,12 @@ flower: # Additional network policies as needed extraNetworkPolicies: [] resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi service: type: ClusterIP @@ -281,12 +281,12 @@ statsd: # Additional network policies as needed extraNetworkPolicies: [] resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi service: extraAnnotations: {} diff --git a/clients/README.md b/clients/README.md new file mode 100644 index 0000000000000..668a98a04c58e --- /dev/null +++ b/clients/README.md @@ -0,0 +1,30 @@ + +Airflow OpenAPI clients +======================= + +Supported languages: + +* [Golang](https://github.com/apache/airflow-client-go) generated through `./gen/go.sh`. + + +## Dependencies + +All client generation scripts use [pre-commit](https://pre-commit.com/#install) +to prepend license header to generated code. diff --git a/clients/gen/common.sh b/clients/gen/common.sh new file mode 100755 index 0000000000000..46fc6455189c6 --- /dev/null +++ b/clients/gen/common.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +OPENAPI_GENERATOR_CLI_VER=4.3.1 +GIT_USER=${GIT_USER:-apache} + +function gen_client { + lang=$1 + shift + docker run --rm \ + -v "${SPEC_PATH}:/spec" \ + -v "${OUTPUT_DIR}:/output" \ + openapitools/openapi-generator-cli:v${OPENAPI_GENERATOR_CLI_VER} \ + generate \ + --input-spec "/spec" \ + --generator-name "${lang}" \ + --git-user-id "${GIT_USER}" \ + --output "/output" "$@" +} diff --git a/clients/gen/go.sh b/clients/gen/go.sh new file mode 100755 index 0000000000000..8c58efae843fb --- /dev/null +++ b/clients/gen/go.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if [ "$#" -ne 2 ]; then + echo "USAGE: $0 SPEC_PATH OUTPUT_DIR" + exit 1 +fi + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +# shellcheck source=./clients/gen/common.sh +source "${SCRIPT_DIR}/common.sh" + +VERSION=1.0.0 +go_config=( + "packageVersion=${VERSION}" + "enumClassPrefix=true" +) + +SPEC_PATH=$(realpath "$1") +if [ ! -d "$2" ]; then + echo "$2 is not a valid directory or does not exist." + exit 1 +fi +OUTPUT_DIR=$(realpath "$2") + +# create openapi ignore file to keep generated code clean +cat < "${OUTPUT_DIR}/.openapi-generator-ignore" +.travis.yml +git_push.sh +EOF + +set -ex +IFS=',' + +SPEC_PATH="${SPEC_PATH}" \ +OUTPUT_DIR="${OUTPUT_DIR}" \ + gen_client go \ + --package-name airflow \ + --git-repo-id airflow/clients/go/airflow \ + --additional-properties "${go_config[*]}" + +# patch generated client to support problem HTTP API +# this patch can be removed after following upstream patch gets merged: +# https://github.com/OpenAPITools/openapi-generator/pull/6793 +cd "${OUTPUT_DIR}" && patch -b <<'EOF' +--- client.go ++++ client.go +@@ -37,7 +37,7 @@ import ( + ) + + var ( +- jsonCheck = regexp.MustCompile(`(?i:(?:application|text)/(?:vnd\.[^;]+\+)?json)`) ++ jsonCheck = regexp.MustCompile(`(?i:(?:application|text)/(?:vnd\.[^;]+\+)?(?:problem\+)?json)`) + xmlCheck = regexp.MustCompile(`(?i:(?:application|text)/xml)`) + ) +EOF + +pushd "${OUTPUT_DIR}" + # prepend license headers + pre-commit run --all-files || true +popd diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index 0e24db7c2d2aa..7ba83a518c903 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -18,8 +18,8 @@ version: "2.2" services: airflow: - stdin_open: true # docker run -i - tty: true # docker run -t + stdin_open: true # docker run -i + tty: true # docker run -t # We need to mount files an directories individually because some files # such apache_airflow.egg-info should not be mounted from host # we only mount those files that it makes sense to edit while developing From d5783245f9ed50e50ad6987228991d42fcb16b7b Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Fri, 3 Jul 2020 11:49:07 -0700 Subject: [PATCH 2/2] move openapi codegen to seperate workflow --- .github/workflows/ci.yml | 26 -------------- .github/workflows/openapi.yml | 35 ++++++++++++++++++ scripts/ci/openapi/client_codegen_diff.sh | 43 +++++++++++++++++++++++ 3 files changed, 78 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/openapi.yml create mode 100755 scripts/ci/openapi/client_codegen_diff.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6061d781bf69..f3b0e351b9a8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -171,32 +171,6 @@ jobs: echo "::set-output name=count::$?" id: trigger-tests - test-openapi-client-generation: - name: "Test OpenAPI client generation" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - with: - fetch-depth: 0 - - name: "Check spec file change" - run: | - URL="https://raw.githubusercontent.com/apache/airflow/master/airflow/api_connexion/openapi/v1.yaml" - curl -sL -o /tmp/current_v1.yaml "${URL}" - if diff /tmp/current_v1.yaml ./airflow/api_connexion/openapi/v1.yaml; then - echo "no openapi spec change detected, going to skip client code gen validation." - else - echo "openapi spec change detected." - echo "::set-env name=API_SPEC_CHANGED::true" - fi - - name: "Generate new clients" - if: success() && env.API_SPEC_CHANGED == 'true' - run: | - mkdir -p ./clients/go/airflow - ./clients/gen/go.sh ./airflow/api_connexion/openapi/v1.yaml ./clients/go/airflow - mkdir -p ./clients/go_master/airflow - ./clients/gen/go.sh /tmp/current_v1.yaml ./clients/go_master/airflow - diff ./clients/go_master/airflow ./clients/go/airflow || true - tests-kubernetes: timeout-minutes: 80 name: "K8s: ${{matrix.kube-mode}} ${{matrix.python-version}} ${{matrix.kubernetes-version}}" diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml new file mode 100644 index 0000000000000..915c40e06b73a --- /dev/null +++ b/.github/workflows/openapi.yml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: OpenAPI Build +on: + push: + branches: ['master'] + pull_request: + branches: ['master'] + +jobs: + test-openapi-client-generation: + name: "Test OpenAPI client generation" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + with: + fetch-depth: 0 + - name: "Generate client codegen diff" + run: ./scripts/ci/openapi/client_codegen_diff.sh diff --git a/scripts/ci/openapi/client_codegen_diff.sh b/scripts/ci/openapi/client_codegen_diff.sh new file mode 100755 index 0000000000000..6b846fe3e2a00 --- /dev/null +++ b/scripts/ci/openapi/client_codegen_diff.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +SCRIPTS_CI_DIR=$(dirname "${BASH_SOURCE[0]}") + +# shellcheck source=scripts/ci/libraries/_initialization.sh +. "${SCRIPTS_CI_DIR}"/../libraries/_initialization.sh +get_environment_for_builds_on_ci + +git remote add target "https://github.com/${CI_TARGET_REPO}" +git fetch target "${CI_TARGET_BRANCH}:${CI_TARGET_BRANCH}" --depth=1 + +echo "Diffing openapi spec against ${CI_TARGET_BRANCH}..." + +SPEC_FILE=airflow/api_connexion/openapi/v1.yaml +if ! git diff --name-only "${CI_TARGET_BRANCH}" HEAD | grep "${SPEC_FILE}" ; then + echo "no openapi spec change detected, going to skip client code gen validation." + exit 0 +fi + +echo "openapi spec change detected. comparing codegen diff..." + +mkdir -p ./clients/go/airflow +./clients/gen/go.sh ./airflow/api_connexion/openapi/v1.yaml ./clients/go/airflow +mkdir -p ./clients/go_target_branch/airflow +git checkout "${CI_TARGET_BRANCH}" ./airflow/api_connexion/openapi/v1.yaml +./clients/gen/go.sh ./airflow/api_connexion/openapi/v1.yaml ./clients/go_target_branch/airflow +diff ./clients/go_target_branch/airflow ./clients/go/airflow || true