From 72030b3f87aa0d03ddf31969ed35d17946e3acf7 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 9 Jul 2019 14:41:25 -0700 Subject: [PATCH 1/7] Fix license check in travis and make it optional --- .travis.yml | 2 +- build.sh | 21 +++++++++++ distribution/pom.xml | 18 +--------- .../generate-license-dependency-reports.py | 2 +- docs/_bin/generate-license.py | 35 ++++++++++++------- 5 files changed, 46 insertions(+), 32 deletions(-) create mode 100755 build.sh diff --git a/.travis.yml b/.travis.yml index 1659b7cb0bdb..90ba01988d83 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,7 +51,7 @@ matrix: - pip3 install wheel # install wheel first explicitly - pip3 install pyyaml install: true - script: MAVEN_OPTS='-Xmx3000m' mvn -DskipTests -Dforbiddenapis.skip=true -Dcheckstyle.skip=true -Dpmd.skip=true -Dmaven.javadoc.skip=true -pl '!benchmarks' -B --fail-at-end clean install -Pdist -Pbundle-contrib-exts + script: ./build.sh # processing module test - env: diff --git a/build.sh b/build.sh new file mode 100755 index 000000000000..59df06b3e61f --- /dev/null +++ b/build.sh @@ -0,0 +1,21 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mkdir -p target + +docs/_bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer && docs/_bin/generate-license.py licenses/APACHE2 licenses.yaml LICENSES.BINARY --dependency-reports target/license-reports + +MAVEN_OPTS='-Xmx3000m' mvn -DskipTests -Dforbiddenapis.skip=true -Dcheckstyle.skip=true -Dpmd.skip=true -Dmaven.javadoc.skip=true -pl '!benchmarks' -B --fail-at-end clean install -Pdist -Pbundle-contrib-exts diff --git a/distribution/pom.xml b/distribution/pom.xml index 2933b02dc650..cbfc489b4fd4 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -131,22 +131,7 @@ - generate-dependency-reports - initialize - - exec - - - ${project.parent.basedir}/docs/_bin/generate-license-dependency-reports.py - - ${project.basedir}/../ - ${project.basedir}/target - --clean-maven-artifact-transfer - - - - - verify-and-generate-license + generate-license initialize exec @@ -156,7 +141,6 @@ ${project.basedir}/../licenses/APACHE2 ${project.basedir}/../licenses.yaml - ${project.basedir}/target/license-reports ${project.basedir}/../LICENSE.BINARY diff --git a/docs/_bin/generate-license-dependency-reports.py b/docs/_bin/generate-license-dependency-reports.py index c691578c50fe..dc8ebdd26e80 100755 --- a/docs/_bin/generate-license-dependency-reports.py +++ b/docs/_bin/generate-license-dependency-reports.py @@ -36,7 +36,7 @@ def generate_report(module_path, report_orig_path, report_out_path): try: print("Generating report for {}".format(module_path)) # This command prints lots of false errors. Here, we redirect stdout and stderr to avoid them. - command = "mvn -Ddependency.locations.enabled=false -Ddependency.details.enabled=false project-info-reports:dependencies" + command = "MAVEN_OPTS='-Xmx1000m' mvn -Ddependency.locations.enabled=false -Ddependency.details.enabled=false project-info-reports:dependencies" subprocess.check_output(command, cwd=module_path, shell=True) command = "cp -r {} {}".format(report_orig_path, report_out_path) subprocess.check_output(command, cwd=module_path, shell=True) diff --git a/docs/_bin/generate-license.py b/docs/_bin/generate-license.py index 1ccc72f5df48..6c5a64330326 100755 --- a/docs/_bin/generate-license.py +++ b/docs/_bin/generate-license.py @@ -20,6 +20,7 @@ import os import sys from html.parser import HTMLParser +import argparse class DependencyReportParser(HTMLParser): @@ -544,16 +545,24 @@ def generate_license(apache_license_v2, license_yaml): print_outfile("") -# TODO: add options: debug mode -if len(sys.argv) != 5: - sys.stderr.write("usage: {} ".format(sys.argv[0])) - sys.exit(1) - -with open(sys.argv[1]) as apache_license_file: - apache_license_v2 = apache_license_file.read() -license_yaml = sys.argv[2] -dependency_reports_root = sys.argv[3] - -with open(sys.argv[4], "w") as outfile: - check_licenses(license_yaml, dependency_reports_root) - generate_license(apache_license_v2, license_yaml) +if __name__ == "__main__": + try: + parser = argparse.ArgumentParser(description='Check and generate license file.') + parser.add_argument('apache_license', metavar='', type=str) + parser.add_argument('license_yaml', metavar='', type=str) + parser.add_argument('out_path', metavar='', type=str) + parser.add_argument('--dependency-reports', dest='dependency_reports_root', type=str, default=None, metavar='') + args = parser.parse_args() + + with open(args.apache_license) as apache_license_file: + apache_license_v2 = apache_license_file.read() + license_yaml = args.license_yaml + dependency_reports_root = args.dependency_reports_root + + with open(args.out_path, "w") as outfile: + if dependency_reports_root is not None: + check_licenses(license_yaml, dependency_reports_root) + generate_license(apache_license_v2, license_yaml) + + except KeyboardInterrupt: + print('Interrupted, closing.') From 6adca7aa3aaa015976ed47906a6a6ed3e25fd73e Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 9 Jul 2019 15:00:01 -0700 Subject: [PATCH 2/7] debug --- docs/_bin/generate-license-dependency-reports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_bin/generate-license-dependency-reports.py b/docs/_bin/generate-license-dependency-reports.py index dc8ebdd26e80..75ec91af49f9 100755 --- a/docs/_bin/generate-license-dependency-reports.py +++ b/docs/_bin/generate-license-dependency-reports.py @@ -37,9 +37,9 @@ def generate_report(module_path, report_orig_path, report_out_path): print("Generating report for {}".format(module_path)) # This command prints lots of false errors. Here, we redirect stdout and stderr to avoid them. command = "MAVEN_OPTS='-Xmx1000m' mvn -Ddependency.locations.enabled=false -Ddependency.details.enabled=false project-info-reports:dependencies" - subprocess.check_output(command, cwd=module_path, shell=True) + subprocess.call(command, cwd=module_path, shell=True) command = "cp -r {} {}".format(report_orig_path, report_out_path) - subprocess.check_output(command, cwd=module_path, shell=True) + subprocess.call(command, cwd=module_path, shell=True) print("Generated report for {} in {}".format(module_path, report_out_path)) except Exception as e: print("Encountered error [{}] when generating report for {}".format(e, module_path)) From 313c373233d91394d35c1b507cf272edd80229ed Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 9 Jul 2019 15:14:22 -0700 Subject: [PATCH 3/7] fix build --- build.sh | 4 ++-- docs/_bin/generate-license-dependency-reports.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index 59df06b3e61f..63e1f874c910 100755 --- a/build.sh +++ b/build.sh @@ -16,6 +16,6 @@ mkdir -p target -docs/_bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer && docs/_bin/generate-license.py licenses/APACHE2 licenses.yaml LICENSES.BINARY --dependency-reports target/license-reports +mvn clean install -DskipTests && docs/_bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer && docs/_bin/generate-license.py licenses/APACHE2 licenses.yaml LICENSES.BINARY --dependency-reports target/license-reports -MAVEN_OPTS='-Xmx3000m' mvn -DskipTests -Dforbiddenapis.skip=true -Dcheckstyle.skip=true -Dpmd.skip=true -Dmaven.javadoc.skip=true -pl '!benchmarks' -B --fail-at-end clean install -Pdist -Pbundle-contrib-exts +MAVEN_OPTS='-Xmx3000m' mvn -DskipTests -Dforbiddenapis.skip=true -Dcheckstyle.skip=true -Dpmd.skip=true -Dmaven.javadoc.skip=true -pl '!benchmarks' -B --fail-at-end install -Pdist -Pbundle-contrib-exts diff --git a/docs/_bin/generate-license-dependency-reports.py b/docs/_bin/generate-license-dependency-reports.py index 75ec91af49f9..dc8ebdd26e80 100755 --- a/docs/_bin/generate-license-dependency-reports.py +++ b/docs/_bin/generate-license-dependency-reports.py @@ -37,9 +37,9 @@ def generate_report(module_path, report_orig_path, report_out_path): print("Generating report for {}".format(module_path)) # This command prints lots of false errors. Here, we redirect stdout and stderr to avoid them. command = "MAVEN_OPTS='-Xmx1000m' mvn -Ddependency.locations.enabled=false -Ddependency.details.enabled=false project-info-reports:dependencies" - subprocess.call(command, cwd=module_path, shell=True) + subprocess.check_output(command, cwd=module_path, shell=True) command = "cp -r {} {}".format(report_orig_path, report_out_path) - subprocess.call(command, cwd=module_path, shell=True) + subprocess.check_output(command, cwd=module_path, shell=True) print("Generated report for {} in {}".format(module_path, report_out_path)) except Exception as e: print("Encountered error [{}] when generating report for {}".format(e, module_path)) From a3545a7f9f6af486bb4dd6e997e4426e5a44b36a Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 9 Jul 2019 15:31:30 -0700 Subject: [PATCH 4/7] too loud maven --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 63e1f874c910..ed691488474f 100755 --- a/build.sh +++ b/build.sh @@ -16,6 +16,6 @@ mkdir -p target -mvn clean install -DskipTests && docs/_bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer && docs/_bin/generate-license.py licenses/APACHE2 licenses.yaml LICENSES.BINARY --dependency-reports target/license-reports +mvn clean install -q -ff -DskipTests && docs/_bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer && docs/_bin/generate-license.py licenses/APACHE2 licenses.yaml LICENSES.BINARY --dependency-reports target/license-reports MAVEN_OPTS='-Xmx3000m' mvn -DskipTests -Dforbiddenapis.skip=true -Dcheckstyle.skip=true -Dpmd.skip=true -Dmaven.javadoc.skip=true -pl '!benchmarks' -B --fail-at-end install -Pdist -Pbundle-contrib-exts From 5d58e7ff080846852dbdd3d7bf18c91d8b5260ae Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 9 Jul 2019 16:23:28 -0700 Subject: [PATCH 5/7] move MAVEN_OPTS to top and add comments --- build.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build.sh b/build.sh index ed691488474f..e3fc5e05c1b8 100755 --- a/build.sh +++ b/build.sh @@ -14,8 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +MAVEN_OPTS='-Xmx3000m' + mkdir -p target +# Generate dependency reports and checks they are valid. mvn clean install -q -ff -DskipTests && docs/_bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer && docs/_bin/generate-license.py licenses/APACHE2 licenses.yaml LICENSES.BINARY --dependency-reports target/license-reports -MAVEN_OPTS='-Xmx3000m' mvn -DskipTests -Dforbiddenapis.skip=true -Dcheckstyle.skip=true -Dpmd.skip=true -Dmaven.javadoc.skip=true -pl '!benchmarks' -B --fail-at-end install -Pdist -Pbundle-contrib-exts +# Build binary distribution. Note that the below command internally runs 'docs/_bin/generate-license.py' without license check which overwrites LICENSES.BINARY file generated by the above command. This must be fine since both commands are supposed to generate the same contents. +mvn -DskipTests -Dforbiddenapis.skip=true -Dcheckstyle.skip=true -Dpmd.skip=true -Dmaven.javadoc.skip=true -pl '!benchmarks' -B --fail-at-end install -Pdist -Pbundle-contrib-exts From e0d8d2e691d60e5a4c6286d501d69cb39dffdac9 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 9 Jul 2019 16:35:55 -0700 Subject: [PATCH 6/7] adjust script --- .travis.yml | 2 +- build.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 90ba01988d83..0b8c577730d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,7 @@ matrix: - sudo apt-get update && sudo apt-get install python3 python3-pip python3-setuptools -y - pip3 install wheel # install wheel first explicitly - pip3 install pyyaml - install: true + install: MAVEN_OPTS='-Xmx3000m' mvn install -q -ff -DskipTests -B script: ./build.sh # processing module test diff --git a/build.sh b/build.sh index e3fc5e05c1b8..bf539b41df84 100755 --- a/build.sh +++ b/build.sh @@ -19,7 +19,9 @@ MAVEN_OPTS='-Xmx3000m' mkdir -p target # Generate dependency reports and checks they are valid. -mvn clean install -q -ff -DskipTests && docs/_bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer && docs/_bin/generate-license.py licenses/APACHE2 licenses.yaml LICENSES.BINARY --dependency-reports target/license-reports +docs/_bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer + +docs/_bin/generate-license.py licenses/APACHE2 licenses.yaml LICENSES.BINARY --dependency-reports target/license-reports # Build binary distribution. Note that the below command internally runs 'docs/_bin/generate-license.py' without license check which overwrites LICENSES.BINARY file generated by the above command. This must be fine since both commands are supposed to generate the same contents. mvn -DskipTests -Dforbiddenapis.skip=true -Dcheckstyle.skip=true -Dpmd.skip=true -Dmaven.javadoc.skip=true -pl '!benchmarks' -B --fail-at-end install -Pdist -Pbundle-contrib-exts From a979085871ed27bb7df43fe74c1731ee7ae2c5af Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 9 Jul 2019 17:31:46 -0700 Subject: [PATCH 7/7] remove mvn option from python script --- docs/_bin/generate-license-dependency-reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_bin/generate-license-dependency-reports.py b/docs/_bin/generate-license-dependency-reports.py index dc8ebdd26e80..c691578c50fe 100755 --- a/docs/_bin/generate-license-dependency-reports.py +++ b/docs/_bin/generate-license-dependency-reports.py @@ -36,7 +36,7 @@ def generate_report(module_path, report_orig_path, report_out_path): try: print("Generating report for {}".format(module_path)) # This command prints lots of false errors. Here, we redirect stdout and stderr to avoid them. - command = "MAVEN_OPTS='-Xmx1000m' mvn -Ddependency.locations.enabled=false -Ddependency.details.enabled=false project-info-reports:dependencies" + command = "mvn -Ddependency.locations.enabled=false -Ddependency.details.enabled=false project-info-reports:dependencies" subprocess.check_output(command, cwd=module_path, shell=True) command = "cp -r {} {}".format(report_orig_path, report_out_path) subprocess.check_output(command, cwd=module_path, shell=True)