Check each grammar on independent CI tasks

.taskcluster.yml

@@ -154,7 +154,7 @@ tasks:
           owner: cdenizet@mozilla.com
           source: ${repository}/raw/${head_rev}/.taskcluster.yml
 
-      - taskId: {$eval: as_slugid("check_tree_sitter_crates")}
+      - taskId: {$eval: as_slugid("check_tree_sitter_mozcpp")}
         dependencies:
            - {$eval: as_slugid("lint_test_task")}
            - {$eval: as_slugid("windows_test_task")}
@@ -171,19 +171,184 @@ tasks:
             - "git clone --quiet ${repository} &&
                cd rust-code-analysis &&
                git -c advice.detachedHead=false checkout ${head_rev} &&
-               ./check-grammars-crates.sh"
+               ./check-grammars-crates.sh tree-sitter-mozcpp"
           cache:
             rust-code-analysis-mozilla-central-repository: /cache
           artifacts:
-            public/json-diffs-and-minimal-tests.tar.gz:
+            public/json-diffs-and-minimal-tests-mozcpp.tar.gz:
               expires: {$fromNow: '2 weeks'}
-              path: /tmp/json-diffs-and-minimal-tests.tar.gz
+              path: /tmp/json-diffs-and-minimal-tests-mozcpp.tar.gz
               type: file
         scopes:
           - "docker-worker:cache:rust-code-analysis-mozilla-central-repository"
         metadata:
-          name: rust-code-analysis check tree-sitter crates
-          description: rust-code-analysis check tree-sitter crates
+          name: rust-code-analysis check tree-sitter-mozcpp
+          description: rust-code-analysis check tree-sitter-mozcpp grammar
+          owner: cdenizet@mozilla.com
+          source: ${repository}/raw/${head_rev}/.taskcluster.yml
+
+      - taskId: {$eval: as_slugid("check_tree_sitter_mozjs")}
+        dependencies:
+           - {$eval: as_slugid("lint_test_task")}
+           - {$eval: as_slugid("windows_test_task")}
+        created: {$fromNow: ''}
+        deadline: {$fromNow: '1 hour'}
+        provisionerId: proj-relman
+        workerType: ci
+        payload:
+          maxRunTime: 3600
+          image: "rust:buster"
+          command:
+            - "/bin/bash"
+            - "-cx"
+            - "git clone --quiet ${repository} &&
+               cd rust-code-analysis &&
+               git -c advice.detachedHead=false checkout ${head_rev} &&
+               ./check-grammars-crates.sh tree-sitter-mozjs"
+          cache:
+            rust-code-analysis-mozilla-central-repository: /cache
+          artifacts:
+            public/json-diffs-and-minimal-tests-mozjs.tar.gz:
+              expires: {$fromNow: '2 weeks'}
+              path: /tmp/json-diffs-and-minimal-tests-mozjs.tar.gz
+              type: file
+        scopes:
+          - "docker-worker:cache:rust-code-analysis-mozilla-central-repository"
+        metadata:
+          name: rust-code-analysis check tree-sitter-mozjs
+          description: rust-code-analysis check tree-sitter-mozjs grammar
+          owner: cdenizet@mozilla.com
+          source: ${repository}/raw/${head_rev}/.taskcluster.yml
+
+      - taskId: {$eval: as_slugid("check_tree_sitter_rust")}
+        dependencies:
+           - {$eval: as_slugid("lint_test_task")}
+           - {$eval: as_slugid("windows_test_task")}
+        created: {$fromNow: ''}
+        deadline: {$fromNow: '1 hour'}
+        provisionerId: proj-relman
+        workerType: ci
+        payload:
+          maxRunTime: 3600
+          image: "rust:buster"
+          command:
+            - "/bin/bash"
+            - "-cx"
+            - "git clone --quiet ${repository} &&
+               cd rust-code-analysis &&
+               git -c advice.detachedHead=false checkout ${head_rev} &&
+               ./check-grammars-crates.sh tree-sitter-rust"
+          cache:
+            rust-code-analysis-mozilla-central-repository: /cache
+          artifacts:
+            public/json-diffs-and-minimal-tests-rust.tar.gz:
+              expires: {$fromNow: '2 weeks'}
+              path: /tmp/json-diffs-and-minimal-tests-rust.tar.gz
+              type: file
+        scopes:
+          - "docker-worker:cache:rust-code-analysis-mozilla-central-repository"
+        metadata:
+          name: rust-code-analysis check tree-sitter-rust
+          description: rust-code-analysis check tree-sitter-rust grammar
+          owner: cdenizet@mozilla.com
+          source: ${repository}/raw/${head_rev}/.taskcluster.yml
+
+      - taskId: {$eval: as_slugid("check_tree_sitter_java")}
+        dependencies:
+           - {$eval: as_slugid("lint_test_task")}
+           - {$eval: as_slugid("windows_test_task")}
+        created: {$fromNow: ''}
+        deadline: {$fromNow: '1 hour'}
+        provisionerId: proj-relman
+        workerType: ci
+        payload:
+          maxRunTime: 3600
+          image: "rust:buster"
+          command:
+            - "/bin/bash"
+            - "-cx"
+            - "git clone --quiet ${repository} &&
+               cd rust-code-analysis &&
+               git -c advice.detachedHead=false checkout ${head_rev} &&
+               ./check-grammars-crates.sh tree-sitter-java"
+          cache:
+            rust-code-analysis-mozilla-central-repository: /cache
+          artifacts:
+            public/json-diffs-and-minimal-tests-java.tar.gz:
+              expires: {$fromNow: '2 weeks'}
+              path: /tmp/json-diffs-and-minimal-tests-java.tar.gz
+              type: file
+        scopes:
+          - "docker-worker:cache:rust-code-analysis-mozilla-central-repository"
+        metadata:
+          name: rust-code-analysis check tree-sitter-java
+          description: rust-code-analysis check tree-sitter-java grammar
+          owner: cdenizet@mozilla.com
+          source: ${repository}/raw/${head_rev}/.taskcluster.yml
+
+      - taskId: {$eval: as_slugid("check_tree_sitter_python")}
+        dependencies:
+           - {$eval: as_slugid("lint_test_task")}
+           - {$eval: as_slugid("windows_test_task")}
+        created: {$fromNow: ''}
+        deadline: {$fromNow: '1 hour'}
+        provisionerId: proj-relman
+        workerType: ci
+        payload:
+          maxRunTime: 3600
+          image: "rust:buster"
+          command:
+            - "/bin/bash"
+            - "-cx"
+            - "git clone --quiet ${repository} &&
+               cd rust-code-analysis &&
+               git -c advice.detachedHead=false checkout ${head_rev} &&
+               ./check-grammars-crates.sh tree-sitter-python"
+          cache:
+            rust-code-analysis-mozilla-central-repository: /cache
+          artifacts:
+            public/json-diffs-and-minimal-tests-python.tar.gz:
+              expires: {$fromNow: '2 weeks'}
+              path: /tmp/json-diffs-and-minimal-tests-python.tar.gz
+              type: file
+        scopes:
+          - "docker-worker:cache:rust-code-analysis-mozilla-central-repository"
+        metadata:
+          name: rust-code-analysis check tree-sitter-python
+          description: rust-code-analysis check tree-sitter-python grammar
+          owner: cdenizet@mozilla.com
+          source: ${repository}/raw/${head_rev}/.taskcluster.yml
+
+      - taskId: {$eval: as_slugid("check_tree_sitter_typescript")}
+        dependencies:
+           - {$eval: as_slugid("lint_test_task")}
+           - {$eval: as_slugid("windows_test_task")}
+        created: {$fromNow: ''}
+        deadline: {$fromNow: '1 hour'}
+        provisionerId: proj-relman
+        workerType: ci
+        payload:
+          maxRunTime: 3600
+          image: "rust:buster"
+          command:
+            - "/bin/bash"
+            - "-cx"
+            - "git clone --quiet ${repository} &&
+               cd rust-code-analysis &&
+               git -c advice.detachedHead=false checkout ${head_rev} &&
+               ./check-grammars-crates.sh tree-sitter-typescript"
+          cache:
+            rust-code-analysis-mozilla-central-repository: /cache
+          artifacts:
+            public/json-diffs-and-minimal-tests-typescript.tar.gz:
+              expires: {$fromNow: '2 weeks'}
+              path: /tmp/json-diffs-and-minimal-tests-typescript.tar.gz
+              type: file
+        scopes:
+          - "docker-worker:cache:rust-code-analysis-mozilla-central-repository"
+        metadata:
+          name: rust-code-analysis check tree-sitter-typescript
+          description: rust-code-analysis check tree-sitter-typescript grammar
           owner: cdenizet@mozilla.com
           source: ${repository}/raw/${head_rev}/.taskcluster.yml
 

check-grammar-crate.py

@@ -41,7 +41,6 @@
 
 # Extensions parsed by each tree-sitter-grammar
 EXTENSIONS = {
-    "tree-sitter-javascript": ["*.js", "*.js2"],
     "tree-sitter-tsx": ["*.tsx"],
     "tree-sitter-typescript": ["*.ts", "*.jsw", "*.jsmw"],
     "tree-sitter-java": ["*.java"],

check-grammars-crates.sh

@@ -3,8 +3,8 @@
 # Stop at the first error
 set -e
 
-# Get all tree-sitter crates from the analyzed branch Cargo.toml
-TS_CRATES=`grep "tree-sitter-*" Cargo.toml | tr -d ' '`
+# Get tree-sitter-grammar
+TS_CRATE=$1
 
 # Disable/Enable CI flag
 RUN_CI="no"
@@ -15,34 +15,29 @@ MASTER_CARGO_TOML="master-cargo.toml"
 # Download master branch Cargo.toml and save it in a temporary file
 wget -LqO - https://raw.githubusercontent.com/mozilla/rust-code-analysis/master/Cargo.toml | tr -d ' ' > $MASTER_CARGO_TOML
 
-# For each tree-sitter crate from the analyzed branch Cargo.toml
-for TS_CRATE in $TS_CRATES
-do
-    # Get the name of the current crate
-    TS_CRATE_NAME=`echo $TS_CRATE | cut -f1 -d "="`
-
-    # Get the crate name from the master branch Cargo.toml
-    MASTER_TS_CRATE_NAME=`grep $TS_CRATE_NAME $MASTER_CARGO_TOML | head -n 1 | cut -f1 -d "="`
-
-    # If the current crate name is not present in master branch, skip to the next crate
-    if [ -z "$MASTER_TS_CRATE_NAME" ]
-    then
-        continue
-    fi
-
-    # Get the same crate from the master branch Cargo.toml
-    MASTER_TS_CRATE=`grep $TS_CRATE $MASTER_CARGO_TOML | head -n 1`
-
-    # If the current crate has been updated, save the crate name and break the loop
-    if [ -z "$MASTER_TS_CRATE" ]
-    then
-        # Enable CI flag
-        RUN_CI="yes"
-        # Name of tree-sitter crate
-        TREE_SITTER_CRATE=$TS_CRATE_NAME
-        break
-    fi
-done
+# Get the name of the current crate
+TS_CRATE_NAME=`echo $TS_CRATE | cut -f1 -d "="`
+
+# Get the crate name from the master branch Cargo.toml
+MASTER_TS_CRATE_NAME=`grep $TS_CRATE_NAME $MASTER_CARGO_TOML | head -n 1 | cut -f1 -d "="`
+
+# If the current crate name is not present in master branch, exit the script
+if [ -z "$MASTER_TS_CRATE_NAME" ]
+then
+    exit 0
+fi
+
+# Get the same crate from the master branch Cargo.toml
+MASTER_TS_CRATE=`grep $TS_CRATE $MASTER_CARGO_TOML | head -n 1`
+
+# If the current crate has been updated, save the crate name
+if [ -z "$MASTER_TS_CRATE" ]
+then
+    # Enable CI flag
+    RUN_CI="yes"
+    # Name of tree-sitter crate
+    TREE_SITTER_CRATE=$TS_CRATE_NAME
+fi
 
 # Remove temporary master branch Cargo.toml file
 rm -rf $MASTER_CARGO_TOML
@@ -95,9 +90,12 @@ if [ "$(ls -A $COMPARE)" ]; then
     # Output directory path
     OUTPUT_DIR=/tmp/output-$TREE_SITTER_CRATE
 
+    # Grammar name (removes tree-sitter- prefix)
+    GRAMMAR_NAME=`echo $TREE_SITTER_CRATE | cut -c 13-`
+
     # Split files into distinct directories depending on
     # their metric differences
     ./split-minimal-tests.py -i $COMPARE -o $OUTPUT_DIR -t $MT_THRESHOLD
 
-    tar -czvf /tmp/json-diffs-and-minimal-tests.tar.gz $COMPARE $OUTPUT_DIR
+    tar -czvf /tmp/json-diffs-and-minimal-tests-$GRAMMAR_NAME.tar.gz $COMPARE $OUTPUT_DIR
 fi