Binary license management system

.travis.yml

@@ -15,12 +15,8 @@
 
 language: java
 
-# On 12-12-2017, Travis updated their trusty image, which caused integration tests to fail.
-# The group: config instructs Travis to use the previous trusty image.
-# Please see https://github.com/apache/incubator-druid/pull/5155 for more information.
-sudo: false
-dist: trusty
-group: deprecated-2017Q4
+sudo: true
+dist: xenial
 
 jdk:
   - openjdk8
@@ -50,6 +46,10 @@ matrix:
     # packaging check
     - env:
         - NAME="packaging check"
+      before_install:
+        - sudo apt-get update && sudo apt-get install python3 python3-pip python3-setuptools -y
+        - pip3 install wheel # install wheel first explicitly
+        - pip3 install pyyaml
       install: true
       script: MAVEN_OPTS='-Xmx3000m' mvn -DskipTests -Dforbiddenapis.skip=true -Dcheckstyle.skip=true -Dpmd.skip=true -Dmaven.javadoc.skip=true -pl '!benchmarks' -B --fail-at-end clean install -Pdist -Pbundle-contrib-exts
 
@@ -117,8 +117,7 @@ matrix:
         - free -m
 
       # run integration tests
-    - sudo: required
-      services:
+    - services:
         - docker
       env:
         - NAME="integration test part 1"
@@ -138,8 +137,7 @@ matrix:
           done
 
       # run integration tests
-    - sudo: required
-      services:
+    - services:
         - docker
       env:
         - NAME="integration test part 2"

distribution/pom.xml

@@ -130,6 +130,37 @@
                                     </arguments>
                                 </configuration>
                             </execution>
+                            <execution>
+                                <id>generate-dependency-reports</id>
+                                <phase>initialize</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>${project.parent.basedir}/docs/_bin/generate-license-dependency-reports.py</executable>
+                                    <arguments>
+                                        <argument>${project.basedir}/../</argument>
+                                        <argument>${project.basedir}/target</argument>
+                                        <argument>--clean-maven-artifact-transfer</argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
+                            <execution>
+                                <id>verify-and-generate-license</id>
+                                <phase>initialize</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>${project.parent.basedir}/docs/_bin/generate-license.py</executable>
+                                    <arguments>
+                                        <argument>${project.basedir}/../licenses/APACHE2</argument>
+                                        <argument>${project.basedir}/../licenses.yaml</argument>
+                                        <argument>${project.basedir}/target/license-reports</argument>
+                                        <argument>${project.basedir}/../LICENSE.BINARY</argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
                             <execution>
                                 <id>pull-deps</id>
                                 <phase>package</phase>

docs/_bin/generate-license-dependency-reports.py

@@ -16,62 +16,87 @@
 # limitations under the License.
 
 import os
+import shutil
 import subprocess
 import sys
 import argparse
+import concurrent.futures
+import time
+import threading
 
 
-existing_jar_dict_notice = {}
+def generate_report(module_path, report_orig_path, report_out_path):
+    is_dir = os.path.isdir(module_path)
+    if not is_dir:
+        print("{} is not a directory".format(module_path))
+        return
 
-def generate_reports(druid_path, tmp_path, exclude_ext):
-    license_main_path =  tmp_path + "/license-reports"
-    license_ext_path = tmp_path + "/license-reports/ext"
-    os.mkdir(license_main_path)
-    os.mkdir(license_ext_path)
+    os.makedirs(report_out_path, exist_ok=True)
+
+    try:
+        print("Generating report for {}".format(module_path))
+        # This command prints lots of false errors. Here, we redirect stdout and stderr to avoid them.
+        command = "mvn -Ddependency.locations.enabled=false -Ddependency.details.enabled=false project-info-reports:dependencies"
+        subprocess.check_output(command, cwd=module_path, shell=True)
+        command = "cp -r {} {}".format(report_orig_path, report_out_path)
+        subprocess.check_output(command, cwd=module_path, shell=True)
+        print("Generated report for {} in {}".format(module_path, report_out_path))
+    except Exception as e:
+        print("Encountered error [{}] when generating report for {}".format(e, module_path))
 
-    print("********** Generating main LICENSE report.... **********")
-    os.chdir(druid_path)
-    command = "mvn -Pdist -Ddependency.locations.enabled=false project-info-reports:dependencies"
-    outstr = subprocess.check_output(command, shell=True).decode('UTF-8')
-    command = "cp -r distribution/target/site {}/site".format(license_main_path)
-    outstr = subprocess.check_output(command, shell=True).decode('UTF-8')
 
-    if exclude_ext:
-        sys.exit()
+def generate_reports(druid_path, tmp_path, exclude_ext, num_threads):
+    tmp_path = os.path.abspath(tmp_path)
+    license_report_root = os.path.join(tmp_path, "license-reports")
+    license_core_path =  os.path.join(license_report_root, "core")
+    license_ext_path = os.path.join(license_report_root, "ext")
+    shutil.rmtree(license_report_root, ignore_errors=True)
+    os.makedirs(license_core_path)
+    os.makedirs(license_ext_path)
+    druid_path = os.path.abspath(druid_path)
 
-    print("********** Generating extension LICENSE reports.... **********")
-    extension_dirs = os.listdir("extensions-core")
-    print("Found {}".format(extension_dirs))
-    for extension_dir in extension_dirs:
-        full_extension_dir = druid_path + "/extensions-core/" + extension_dir
-        if not os.path.isdir(full_extension_dir):
-            print("{} is not a directory".format(full_extension_dir))
-            continue
+    script_args = [(druid_path, os.path.join(druid_path, "distribution", "target", "site"), license_core_path)]
 
-        print("--- Generating report for {}... ---".format(extension_dir))
+    if not exclude_ext:
+        extensions_core_path = os.path.join(druid_path, "extensions-core")
+        extension_dirs = os.listdir(extensions_core_path)
+        print("Found {} extensions".format(len(extension_dirs)))
+        for extension_dir in extension_dirs:
+            print("extension dir: {}".format(extension_dir))
+            extension_path = os.path.join(extensions_core_path, extension_dir)
+            if not os.path.isdir(extension_path):
+                print("{} is not a directory".format(extension_path))
+                continue
 
-        extension_report_dir = "{}/{}".format(license_ext_path, extension_dir)
-        os.mkdir(extension_report_dir)
-        prev_work_dir = os.getcwd()
-        os.chdir(full_extension_dir)
+            extension_report_dir = "{}/{}".format(license_ext_path, extension_dir)
+            script_args.append((extension_path, os.path.join(extension_path, "target", "site"), extension_report_dir))
+
+    print("Generating dependency reports")
 
-        try:
-            command = "mvn -Ddependency.locations.enabled=false project-info-reports:dependencies"
-            outstr = subprocess.check_output(command, shell=True).decode('UTF-8')
-            command = "cp -r target/site {}/site".format(extension_report_dir)
-            outstr = subprocess.check_output(command, shell=True).decode('UTF-8')
-        except:
-            print("Encountered error when generating report for: " + extension_dir)
+    if num_threads > 1:
+        with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
+            for module_path, report_orig_path, report_out_path in script_args:
+                executor.submit(generate_report, module_path, report_orig_path, report_out_path)
+    else:
+        for module_path, report_orig_path, report_out_path in script_args:
+            generate_report(module_path, report_orig_path, report_out_path)
 
-        os.chdir(prev_work_dir)
 
 if __name__ == "__main__":
     try:
         parser = argparse.ArgumentParser(description='Generating dependency reports.')
         parser.add_argument('druid_path', metavar='<Druid source path>', type=str)
         parser.add_argument('tmp_path', metavar='<Full tmp path>', type=str)
         parser.add_argument('--exclude-extension', dest='exclude_ext', action='store_const', const=True, default=False, help="Exclude extension report")
+        parser.add_argument('--clean-maven-artifact-transfer', dest='clean_mvn_artifact_transfer', action='store_const', const=True, default=False, help="Clean maven-artifact-transfer before generating dependency reports")
+        parser.add_argument('--parallel', dest='num_threads', type=int, default=1, help='Number of threads for generating reports')
         args = parser.parse_args()
-        generate_reports(args.druid_path, args.tmp_path, args.exclude_ext)
+
+        # The default maven-artifact-transfer in Travis is currently corrupted. Set the below argument properly to remove the corrupted one.
+        if args.clean_mvn_artifact_transfer:
+            command = "rm -rf ~/.m2/repository/org/apache/maven/shared/maven-artifact-transfer"
+            subprocess.check_call(command, shell=True)
+
+        generate_reports(args.druid_path, args.tmp_path, args.exclude_ext, args.num_threads)
     except KeyboardInterrupt:
         print('Interrupted, closing.')