Create new google-cloud-conformance-tests module

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "google-cloud-testing/google-cloud-conformance-tests/conformance-tests"]
+	path = google-cloud-testing/google-cloud-conformance-tests/conformance-tests
+	url = git@github.com:googleapis/conformance-tests.git

google-cloud-clients/google-cloud-firestore/pom.xml

@@ -108,32 +108,11 @@
       <classifier>testlib</classifier>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-conformance-tests</artifactId>
+    </dependency>
   </dependencies>
-  <build>
-    <plugins>
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>build-helper-maven-plugin</artifactId>
-        <version>3.0.0</version>
-        <executions>
-          <execution>
-            <id>add-test-resource</id>
-            <phase>generate-test-resources</phase>
-            <goals>
-              <goal>add-test-resource</goal>
-            </goals>
-            <configuration>
-              <resources>
-                <resource>
-                  <directory>src/test/proto</directory>
-                </resource>
-              </resources>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-    </plugins>
-  </build>
 
   <reporting>
     <plugins>

google-cloud-clients/google-cloud-firestore/src/test/java/com/google/cloud/firestore/ConformanceConversions.java

@@ -17,7 +17,7 @@
 package com.google.cloud.firestore;
 
 import com.google.cloud.Timestamp;
-import com.google.cloud.firestore.conformance.TestDefinition;
+import com.google.cloud.conformance.firestore.v1.TestDefinition;
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
 import java.lang.reflect.Type;

google-cloud-clients/google-cloud-firestore/src/test/java/com/google/cloud/firestore/ConformanceTest.java

@@ -16,6 +16,7 @@
 
 package com.google.cloud.firestore;
 
+import static com.google.cloud.conformance.ConformanceTestLocator.newMatchPattern;
 import static com.google.cloud.firestore.ConformanceConversions.convertInput;
 import static com.google.cloud.firestore.ConformanceConversions.convertPath;
 import static com.google.cloud.firestore.ConformanceConversions.convertPaths;
@@ -35,23 +36,25 @@
 import com.google.api.gax.rpc.ServerStreamingCallable;
 import com.google.api.gax.rpc.UnaryCallable;
 import com.google.cloud.Timestamp;
+import com.google.cloud.conformance.ConformanceTestLocator;
+import com.google.cloud.conformance.ConformanceTestLocator.MatchPattern;
+import com.google.cloud.conformance.firestore.v1.TestDefinition;
+import com.google.cloud.conformance.firestore.v1.TestDefinition.Clause;
+import com.google.cloud.conformance.firestore.v1.TestDefinition.CreateTest;
+import com.google.cloud.conformance.firestore.v1.TestDefinition.Cursor;
+import com.google.cloud.conformance.firestore.v1.TestDefinition.DeleteTest;
+import com.google.cloud.conformance.firestore.v1.TestDefinition.DocChange;
+import com.google.cloud.conformance.firestore.v1.TestDefinition.DocChange.Kind;
+import com.google.cloud.conformance.firestore.v1.TestDefinition.DocSnapshot;
+import com.google.cloud.conformance.firestore.v1.TestDefinition.GetTest;
+import com.google.cloud.conformance.firestore.v1.TestDefinition.ListenTest;
+import com.google.cloud.conformance.firestore.v1.TestDefinition.OrderBy;
+import com.google.cloud.conformance.firestore.v1.TestDefinition.SetTest;
+import com.google.cloud.conformance.firestore.v1.TestDefinition.Snapshot;
+import com.google.cloud.conformance.firestore.v1.TestDefinition.UpdatePathsTest;
+import com.google.cloud.conformance.firestore.v1.TestDefinition.UpdateTest;
+import com.google.cloud.conformance.firestore.v1.TestDefinition.Where;
 import com.google.cloud.firestore.Query.Direction;
-import com.google.cloud.firestore.conformance.TestDefinition;
-import com.google.cloud.firestore.conformance.TestDefinition.Clause;
-import com.google.cloud.firestore.conformance.TestDefinition.CreateTest;
-import com.google.cloud.firestore.conformance.TestDefinition.Cursor;
-import com.google.cloud.firestore.conformance.TestDefinition.DeleteTest;
-import com.google.cloud.firestore.conformance.TestDefinition.DocChange;
-import com.google.cloud.firestore.conformance.TestDefinition.DocChange.Kind;
-import com.google.cloud.firestore.conformance.TestDefinition.DocSnapshot;
-import com.google.cloud.firestore.conformance.TestDefinition.GetTest;
-import com.google.cloud.firestore.conformance.TestDefinition.ListenTest;
-import com.google.cloud.firestore.conformance.TestDefinition.OrderBy;
-import com.google.cloud.firestore.conformance.TestDefinition.SetTest;
-import com.google.cloud.firestore.conformance.TestDefinition.Snapshot;
-import com.google.cloud.firestore.conformance.TestDefinition.UpdatePathsTest;
-import com.google.cloud.firestore.conformance.TestDefinition.UpdateTest;
-import com.google.cloud.firestore.conformance.TestDefinition.Where;
 import com.google.cloud.firestore.spi.v1.FirestoreRpc;
 import com.google.common.base.Preconditions;
 import com.google.firestore.v1.BatchGetDocumentsRequest;
@@ -63,8 +66,11 @@
 import com.google.firestore.v1.Value;
 import com.google.protobuf.AbstractMessage;
 import com.google.protobuf.Message;
+import com.google.protobuf.util.JsonFormat;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -94,9 +100,6 @@
 @RunWith(AllTests.class)
 public class ConformanceTest {
 
-  private static final String TEST_FILE =
-      "/com/google/cloud/firestore/conformance/test-suite.binproto";
-
   /** Excluded tests by test description. */
   private static final Set<String> excludedTests = Collections.emptySet();
 
@@ -109,7 +112,7 @@ public class ConformanceTest {
   public ConformanceTest() {}
 
   /** Generate the test suite based on the tests defined in test_data.binprotos. */
-  public static TestSuite suite() throws IOException {
+  public static TestSuite suite() throws Exception {
     TestSuite suite = new TestSuite();
     final TestDefinition.TestSuite testSuite = parseTests();
     for (Test test : initTests(testSuite)) {
@@ -134,9 +137,24 @@ private static List<Test> initTests(TestDefinition.TestSuite testSuite) {
   }
 
   /** Reads the test definition from the Proto file. */
-  private static TestDefinition.TestSuite parseTests() throws IOException {
-    final InputStream is = ConformanceTest.class.getResourceAsStream(TEST_FILE);
-    return TestDefinition.TestSuite.parseFrom(is);
+  private static TestDefinition.TestSuite parseTests() throws IOException, URISyntaxException {
+    final ClassLoader cl = Thread.currentThread().getContextClassLoader();
+    final MatchPattern matchPattern =
+        newMatchPattern("com/google/cloud/conformance/firestore/v1", ".json");
+    final List<String> jsonResources =
+        ConformanceTestLocator.findAllResourcePaths(matchPattern, cl);
+
+    final TestDefinition.TestSuite.Builder suite = TestDefinition.TestSuite.newBuilder();
+
+    for (String jsonResource : jsonResources) {
+      final InputStream is = cl.getResourceAsStream(jsonResource);
+      Assert.assertNotNull(String.format("Unable to load test definition: '%s'", jsonResource), is);
+      final InputStreamReader reader = new InputStreamReader(is);
+      final TestDefinition.TestSuite.Builder testBuilder = TestDefinition.TestSuite.newBuilder();
+      JsonFormat.parser().merge(reader, testBuilder);
+      suite.addAllTests(testBuilder.build().getTestsList());
+    }
+    return suite.build();
   }
 
   /** Returns the test case for the provided test definition. */

google-cloud-clients/pom.xml

@@ -420,6 +420,12 @@
         <artifactId>checker-compat-qual</artifactId>
         <version>2.5.5</version>
       </dependency>
+      <dependency>
+        <groupId>com.google.cloud</groupId>
+        <artifactId>google-cloud-conformance-tests</artifactId>
+        <version>0.1.0-SNAPSHOT</version><!-- {x-version-update:google-cloud-conformance-tests:current} -->
+        <scope>test</scope>
+      </dependency>
     </dependencies>
   </dependencyManagement>
   <modules>

google-cloud-testing/google-cloud-conformance-tests/copyright-header.txt

@@ -0,0 +1,15 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */

google-cloud-testing/google-cloud-conformance-tests/generate-conformance-tests.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+# Copyright 2019 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit -o nounset -o pipefail
+
+function generateFirestore() {
+
+  rm generate.log 2> /dev/null || true
+
+  local javaPackage="com/google/cloud/conformance/firestore/v1"
+
+  msg "Cleaning existing generated protos"
+  rm -rf src/main/java/${javaPackage}/*
+  msg "Cleaning existing test definitions"
+  rm -rf src/main/resources/${javaPackage}/*
+  msg "Cleaning existing proto files"
+  rm -rf src/main/proto/google/cloud/conformance/firestore/v1/*
+
+  msg "Copying new proto files"
+  cp -R -t src/main/proto/ conformance-tests/firestore/v1/proto/*
+  msg "Copying new test definitions"
+  cp -R -t "src/main/resources/${javaPackage}/" conformance-tests/firestore/v1/testcase/*
+
+  msg "Generating protos"
+  mvn -Pgen-conformance-protos clean verify > generate.log 2>&1
+
+  msg "Adding copyright header to generated sources"
+  ## java classes generated from protoc do not include the copyright header
+  ## prepend it to the generated files
+  local moduleDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+  pushd target/generated-sources/protobuf/java/${javaPackage}
+  for f in $(find ./ -type f -name '*.java'); do
+    echo "Processing $f"
+    mv ${f} ${f}.tmp
+    cat ${moduleDir}/copyright-header.txt > ${f}
+    cat ${f}.tmp >> ${f}
+    rm ${f}.tmp
+  done
+  popd
+
+  ## move generated proto class(es) into the main src root
+  cp -R -t src/main/java/${javaPackage} target/generated-sources/protobuf/java/${javaPackage}/*
+
+  ## cleanup any generated files that may have not been moved over
+  mvn clean > /dev/null 2>&1
+
+  msg "Building module"
+  ## ensure building of the module still works
+  mvn clean package
+
+}
+
+function main() {
+  generateFirestore
+}
+
+function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;}
+function msg { println "$*" >&2 ;}
+function println { printf '%s\n' "$(now) $*" ;}
+
+main

google-cloud-testing/google-cloud-conformance-tests/pom.xml

@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>google-cloud-conformance-tests</artifactId>
+  <!-- TODO(benwhitehead): What should this version actually be? -->
+  <version>0.1.0-SNAPSHOT</version><!-- {x-version-update:google-cloud-conformance-tests:current} -->
+  <packaging>jar</packaging>
+
+  <parent>
+    <groupId>com.google.cloud</groupId>
+    <artifactId>google-cloud-testing</artifactId>
+    <version>0.94.1-alpha-SNAPSHOT</version><!-- {x-version-update:google-cloud-testing:current} -->
+  </parent>
+
+  <properties>
+    <protobuf.version>3.7.1</protobuf.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.truth</groupId>
+      <artifactId>truth</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.google.api.grpc</groupId>
+      <artifactId>proto-google-cloud-firestore-v1</artifactId>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <extensions>
+      <extension>
+        <groupId>kr.motd.maven</groupId>
+        <artifactId>os-maven-plugin</artifactId>
+        <version>1.6.2</version>
+      </extension>
+    </extensions>
+    <plugins>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>build-helper-maven-plugin</artifactId>
+        <version>3.0.0</version>
+        <executions>
+          <execution>
+            <id>add-main-resource</id>
+            <phase>generate-resources</phase>
+            <goals>
+              <goal>add-resource</goal>
+            </goals>
+            <configuration>
+              <resources>
+                <resource>
+                  <directory>src/main/proto</directory>
+                </resource>
+              </resources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+  <profiles>
+    <profile>
+      <id>gen-conformance-protos</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.xolstice.maven.plugins</groupId>
+            <artifactId>protobuf-maven-plugin</artifactId>
+            <version>0.6.1</version>
+            <executions>
+              <execution>
+                <id>generate-main</id>
+                <goals>
+                  <goal>compile</goal>
+                </goals>
+                <phase>generate-sources</phase>
+              </execution>
+              <execution>
+                <id>generate-test</id>
+                <goals>
+                  <goal>test-compile</goal>
+                </goals>
+                <phase>generate-test-sources</phase>
+              </execution>
+            </executions>
+            <configuration>
+              <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+            </configuration>
+          </plugin>
+          <plugin>
+            <groupId>com.coveo</groupId>
+            <artifactId>fmt-maven-plugin</artifactId>
+            <version>2.9</version>
+            <executions>
+              <execution>
+                <id>format-main</id>
+                <goals>
+                  <goal>format</goal>
+                </goals>
+                <phase>process-sources</phase>
+              </execution>
+              <execution>
+                <id>format-test</id>
+                <goals>
+                  <goal>format</goal>
+                </goals>
+                <phase>process-test-sources</phase>
+              </execution>
+            </executions>
+            <configuration>
+              <additionalSourceDirectories>
+                <directory>target/generated-sources/protobuf/java</directory>
+              </additionalSourceDirectories>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
+
+</project>