From 25a7e86e0772b4a5153a8abf414dab2f90f0b857 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Mon, 24 Apr 2023 16:05:36 -0400 Subject: [PATCH 1/3] Add artifact validation to run_rc_validation --- release/src/main/scripts/run_rc_validation.sh | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/release/src/main/scripts/run_rc_validation.sh b/release/src/main/scripts/run_rc_validation.sh index 997525ef8006..198a77af1685 100755 --- a/release/src/main/scripts/run_rc_validation.sh +++ b/release/src/main/scripts/run_rc_validation.sh @@ -100,6 +100,8 @@ HUB_ARTIFACTS_NAME=hub-linux-amd64-${HUB_VERSION} BACKUP_BASHRC=.bashrc_backup_$(date +"%Y%m%d%H%M%S") BACKUP_M2=settings_backup_$(date +"%Y%m%d%H%M%S").xml declare -a PYTHON_VERSIONS_TO_VALIDATE=("python3.8") +# Used to define which artifacts should be present in the generated apache repository. Please add/subtract artifacts as they are published/no longer published. +ARTIFACTS=( "beam-examples-java-cdap-hubspot" "beam-examples-java-cdap-salesforce" "beam-examples-java-cdap-servicenow" "beam-examples-java-cdap-zendesk" "beam-examples-java-cdap" "beam-examples-java-twitter" "beam-examples-java" "beam-examples-kotlin" "beam-examples-multi-language" "beam-model-fn-execution" "beam-model-job-management" "beam-model-pipeline" "beam-runners-core-construction-java" "beam-runners-core-java" "beam-runners-direct-java" "beam-runners-extensions-java-metrics" "beam-runners-flink-1.12-job-server" "beam-runners-flink-1.12" "beam-runners-flink-1.13-job-server" "beam-runners-flink-1.13" "beam-runners-flink-1.14-job-server" "beam-runners-flink-1.14" "beam-runners-flink-1.15-job-server" "beam-runners-flink-1.15" "beam-runners-flink-1.16-job-server" "beam-runners-flink-1.16" "beam-runners-google-cloud-dataflow-java-legacy-worker" "beam-runners-google-cloud-dataflow-java" "beam-runners-java-fn-execution" "beam-runners-java-job-service" "beam-runners-jet" "beam-runners-local-java-core" "beam-runners-portability-java" "beam-runners-samza-job-server" "beam-runners-samza" "beam-runners-spark-3-job-server" "beam-runners-spark-3" "beam-runners-twister2" "beam-sdks-java-bom" "beam-sdks-java-core" "beam-sdks-java-expansion-service-app" "beam-sdks-java-expansion-service" "beam-sdks-java-extensions-arrow" "beam-sdks-java-extensions-avro" "beam-sdks-java-extensions-euphoria" "beam-sdks-java-extensions-google-cloud-platform-core" "beam-sdks-java-extensions-join-library" "beam-sdks-java-extensions-json-jackson" "beam-sdks-java-extensions-kryo" "beam-sdks-java-extensions-ml" "beam-sdks-java-extensions-protobuf" "beam-sdks-java-extensions-python" "beam-sdks-java-extensions-sbe" "beam-sdks-java-extensions-schemaio-expansion-service" "beam-sdks-java-extensions-sketching" "beam-sdks-java-extensions-sorter" "beam-sdks-java-extensions-sql-datacatalog" "beam-sdks-java-extensions-sql-expansion-service" "beam-sdks-java-extensions-sql-hcatalog" "beam-sdks-java-extensions-sql-jdbc" "beam-sdks-java-extensions-sql-udf" "beam-sdks-java-extensions-sql-zetasql" "beam-sdks-java-extensions-sql" "beam-sdks-java-extensions-timeseries" "beam-sdks-java-extensions-zetasketch" "beam-sdks-java-fn-execution" "beam-sdks-java-google-cloud-platform-bom" "beam-sdks-java-harness" "beam-sdks-java-io-amazon-web-services" "beam-sdks-java-io-amazon-web-services2" "beam-sdks-java-io-amqp" "beam-sdks-java-io-azure" "beam-sdks-java-io-cassandra" "beam-sdks-java-io-cdap" "beam-sdks-java-io-clickhouse" "beam-sdks-java-io-common" "beam-sdks-java-io-contextualtextio" "beam-sdks-java-io-csv" "beam-sdks-java-io-debezium-expansion-service" "beam-sdks-java-io-debezium" "beam-sdks-java-io-elasticsearch" "beam-sdks-java-io-expansion-service" "beam-sdks-java-io-file-schema-transform" "beam-sdks-java-io-google-cloud-platform-expansion-service" "beam-sdks-java-io-google-cloud-platform" "beam-sdks-java-io-hadoop-common" "beam-sdks-java-io-hadoop-file-system" "beam-sdks-java-io-hadoop-format" "beam-sdks-java-io-hbase" "beam-sdks-java-io-hcatalog" "beam-sdks-java-io-influxdb" "beam-sdks-java-io-jdbc" "beam-sdks-java-io-jms" "beam-sdks-java-io-kafka" "beam-sdks-java-io-kinesis-expansion-service" "beam-sdks-java-io-kinesis" "beam-sdks-java-io-kudu" "beam-sdks-java-io-mongodb" "beam-sdks-java-io-mqtt" "beam-sdks-java-io-neo4j" "beam-sdks-java-io-parquet" "beam-sdks-java-io-pulsar" "beam-sdks-java-io-rabbitmq" "beam-sdks-java-io-redis" "beam-sdks-java-io-singlestore" "beam-sdks-java-io-snowflake-expansion-service" "beam-sdks-java-io-snowflake" "beam-sdks-java-io-solr" "beam-sdks-java-io-sparkreceiver-2" "beam-sdks-java-io-splunk" "beam-sdks-java-io-synthetic" "beam-sdks-java-io-thrift" "beam-sdks-java-io-tika" "beam-sdks-java-io-xml" "beam-sdks-java-maven-archetypes-examples" "beam-sdks-java-maven-archetypes-gcp-bom-examples" "beam-sdks-java-maven-archetypes-starter" "beam-sdks-java-nexmark" "beam-sdks-java-test-utils" "beam-sdks-java-tpcds" ) echo "" echo "====================Checking Environment & Variables=================" @@ -123,6 +125,53 @@ if [[ $confirmation != "y" ]]; then exit fi +echo "----------------- Checking published Java artifacts (should take ~1 minute) -----------------" +FAILED=() +for i in "${ARTIFACTS[@]}" +do + curl "${REPO_URL}/org/apache/beam/${i}/${RELEASE_VER}" -f || FAILED+=($i) +done +if [ ${#FAILED[@]} != 0 ]; +then + echo "Failed to find the following artifacts:" + for i in "${FAILED[@]}" + do + echo "Artifact: ${i} - url: ${REPO_URL}/org/apache/beam/${i}/${RELEASE_VER}" + done + echo "If these artifacts should no longer be published, remove them from the ARTIFACTS variable in run_rc_validation.sh and try again." + echo "Please check any changes back in to master." + exit 1 +fi +response=$(curl "${REPO_URL}/org/apache/beam/" | grep -Eo "${REPO_URL}/\S+?\"") +published=($response) +for p in "${published[@]}" +do + in_artifacts=0 + for a in "${ARTIFACTS[@]}" + do + if [ "${REPO_URL}/org/apache/beam/${a}/\"" == $p ] + then + in_artifacts=1 + fi + done + + if [ $in_artifacts == 0 ] + then + FAILED+=($a) + fi +done +if [ ${#FAILED[@]} != 0 ]; +then + echo "The following artifacts were published but maybe should not have been:" + for i in "${FAILED[@]}" + do + echo "Artifact: ${i} - url: ${REPO_URL}/org/apache/beam/${i}/${RELEASE_VER}" + done + echo "If these artifacts should have been published (which is likely), add them from to ARTIFACTS variable in run_rc_validation.sh and try again." + echo "Please check any changes back in to master." + exit 1 +fi + echo "----------------- Checking git -----------------" if [[ -z ${GITHUB_TOKEN} ]]; then echo "Error: A Github personal access token is required to perform git push " From 190507c2a47f55d4d0af619cf1b2f43caaf6222e Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Wed, 26 Apr 2023 15:42:57 -0400 Subject: [PATCH 2/3] config file for artifacts --- release/src/main/scripts/java_artifacts.txt | 120 ++++++++++++++++++ release/src/main/scripts/run_rc_validation.sh | 8 +- 2 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 release/src/main/scripts/java_artifacts.txt diff --git a/release/src/main/scripts/java_artifacts.txt b/release/src/main/scripts/java_artifacts.txt new file mode 100644 index 000000000000..3fe333bf074d --- /dev/null +++ b/release/src/main/scripts/java_artifacts.txt @@ -0,0 +1,120 @@ +beam-examples-java-cdap-hubspot +beam-examples-java-cdap-salesforce +beam-examples-java-cdap-servicenow +beam-examples-java-cdap-zendesk +beam-examples-java-cdap +beam-examples-java-twitter +beam-examples-java +beam-examples-kotlin +beam-examples-multi-language +beam-model-fn-execution +beam-model-job-management +beam-model-pipeline +beam-runners-core-construction-java +beam-runners-core-java +beam-runners-direct-java +beam-runners-extensions-java-metrics +beam-runners-flink-1.12-job-server +beam-runners-flink-1.12 +beam-runners-flink-1.13-job-server +beam-runners-flink-1.13 +beam-runners-flink-1.14-job-server +beam-runners-flink-1.14 +beam-runners-flink-1.15-job-server +beam-runners-flink-1.15 +beam-runners-flink-1.16-job-server +beam-runners-flink-1.16 +beam-runners-google-cloud-dataflow-java-legacy-worker +beam-runners-google-cloud-dataflow-java +beam-runners-java-fn-execution +beam-runners-java-job-service +beam-runners-jet +beam-runners-local-java-core +beam-runners-portability-java +beam-runners-samza-job-server +beam-runners-samza +beam-runners-spark-3-job-server +beam-runners-spark-3 +beam-runners-twister2 +beam-sdks-java-bom +beam-sdks-java-core +beam-sdks-java-expansion-service-app +beam-sdks-java-expansion-service +beam-sdks-java-extensions-arrow +beam-sdks-java-extensions-avro +beam-sdks-java-extensions-euphoria +beam-sdks-java-extensions-google-cloud-platform-core +beam-sdks-java-extensions-join-library +beam-sdks-java-extensions-json-jackson +beam-sdks-java-extensions-kryo +beam-sdks-java-extensions-ml +beam-sdks-java-extensions-protobuf +beam-sdks-java-extensions-python +beam-sdks-java-extensions-sbe +beam-sdks-java-extensions-schemaio-expansion-service +beam-sdks-java-extensions-sketching +beam-sdks-java-extensions-sorter +beam-sdks-java-extensions-sql-datacatalog +beam-sdks-java-extensions-sql-expansion-service +beam-sdks-java-extensions-sql-hcatalog +beam-sdks-java-extensions-sql-jdbc +beam-sdks-java-extensions-sql-udf +beam-sdks-java-extensions-sql-zetasql +beam-sdks-java-extensions-sql +beam-sdks-java-extensions-timeseries +beam-sdks-java-extensions-zetasketch +beam-sdks-java-fn-execution +beam-sdks-java-google-cloud-platform-bom +beam-sdks-java-harness +beam-sdks-java-io-amazon-web-services +beam-sdks-java-io-amazon-web-services2 +beam-sdks-java-io-amqp +beam-sdks-java-io-azure +beam-sdks-java-io-cassandra +beam-sdks-java-io-cdap +beam-sdks-java-io-clickhouse +beam-sdks-java-io-common +beam-sdks-java-io-contextualtextio +beam-sdks-java-io-csv +beam-sdks-java-io-debezium-expansion-service +beam-sdks-java-io-debezium +beam-sdks-java-io-elasticsearch +beam-sdks-java-io-expansion-service +beam-sdks-java-io-file-schema-transform +beam-sdks-java-io-google-cloud-platform-expansion-service +beam-sdks-java-io-google-cloud-platform +beam-sdks-java-io-hadoop-common +beam-sdks-java-io-hadoop-file-system +beam-sdks-java-io-hadoop-format +beam-sdks-java-io-hbase +beam-sdks-java-io-hcatalog +beam-sdks-java-io-influxdb +beam-sdks-java-io-jdbc +beam-sdks-java-io-jms +beam-sdks-java-io-kafka +beam-sdks-java-io-kinesis-expansion-service +beam-sdks-java-io-kinesis +beam-sdks-java-io-kudu +beam-sdks-java-io-mongodb +beam-sdks-java-io-mqtt +beam-sdks-java-io-neo4j +beam-sdks-java-io-parquet +beam-sdks-java-io-pulsar +beam-sdks-java-io-rabbitmq +beam-sdks-java-io-redis +beam-sdks-java-io-singlestore +beam-sdks-java-io-snowflake-expansion-service +beam-sdks-java-io-snowflake +beam-sdks-java-io-solr +beam-sdks-java-io-sparkreceiver-2 +beam-sdks-java-io-splunk +beam-sdks-java-io-synthetic +beam-sdks-java-io-thrift +beam-sdks-java-io-tika +beam-sdks-java-io-xml +beam-sdks-java-maven-archetypes-examples +beam-sdks-java-maven-archetypes-gcp-bom-examples +beam-sdks-java-maven-archetypes-starter +beam-sdks-java-nexmark +beam-sdks-java-test-utils +beam-sdks-java-tpcds \ No newline at end of file diff --git a/release/src/main/scripts/run_rc_validation.sh b/release/src/main/scripts/run_rc_validation.sh index 198a77af1685..39f57f5aa35e 100755 --- a/release/src/main/scripts/run_rc_validation.sh +++ b/release/src/main/scripts/run_rc_validation.sh @@ -101,8 +101,8 @@ BACKUP_BASHRC=.bashrc_backup_$(date +"%Y%m%d%H%M%S") BACKUP_M2=settings_backup_$(date +"%Y%m%d%H%M%S").xml declare -a PYTHON_VERSIONS_TO_VALIDATE=("python3.8") # Used to define which artifacts should be present in the generated apache repository. Please add/subtract artifacts as they are published/no longer published. -ARTIFACTS=( "beam-examples-java-cdap-hubspot" "beam-examples-java-cdap-salesforce" "beam-examples-java-cdap-servicenow" "beam-examples-java-cdap-zendesk" "beam-examples-java-cdap" "beam-examples-java-twitter" "beam-examples-java" "beam-examples-kotlin" "beam-examples-multi-language" "beam-model-fn-execution" "beam-model-job-management" "beam-model-pipeline" "beam-runners-core-construction-java" "beam-runners-core-java" "beam-runners-direct-java" "beam-runners-extensions-java-metrics" "beam-runners-flink-1.12-job-server" "beam-runners-flink-1.12" "beam-runners-flink-1.13-job-server" "beam-runners-flink-1.13" "beam-runners-flink-1.14-job-server" "beam-runners-flink-1.14" "beam-runners-flink-1.15-job-server" "beam-runners-flink-1.15" "beam-runners-flink-1.16-job-server" "beam-runners-flink-1.16" "beam-runners-google-cloud-dataflow-java-legacy-worker" "beam-runners-google-cloud-dataflow-java" "beam-runners-java-fn-execution" "beam-runners-java-job-service" "beam-runners-jet" "beam-runners-local-java-core" "beam-runners-portability-java" "beam-runners-samza-job-server" "beam-runners-samza" "beam-runners-spark-3-job-server" "beam-runners-spark-3" "beam-runners-twister2" "beam-sdks-java-bom" "beam-sdks-java-core" "beam-sdks-java-expansion-service-app" "beam-sdks-java-expansion-service" "beam-sdks-java-extensions-arrow" "beam-sdks-java-extensions-avro" "beam-sdks-java-extensions-euphoria" "beam-sdks-java-extensions-google-cloud-platform-core" "beam-sdks-java-extensions-join-library" "beam-sdks-java-extensions-json-jackson" "beam-sdks-java-extensions-kryo" "beam-sdks-java-extensions-ml" "beam-sdks-java-extensions-protobuf" "beam-sdks-java-extensions-python" "beam-sdks-java-extensions-sbe" "beam-sdks-java-extensions-schemaio-expansion-service" "beam-sdks-java-extensions-sketching" "beam-sdks-java-extensions-sorter" "beam-sdks-java-extensions-sql-datacatalog" "beam-sdks-java-extensions-sql-expansion-service" "beam-sdks-java-extensions-sql-hcatalog" "beam-sdks-java-extensions-sql-jdbc" "beam-sdks-java-extensions-sql-udf" "beam-sdks-java-extensions-sql-zetasql" "beam-sdks-java-extensions-sql" "beam-sdks-java-extensions-timeseries" "beam-sdks-java-extensions-zetasketch" "beam-sdks-java-fn-execution" "beam-sdks-java-google-cloud-platform-bom" "beam-sdks-java-harness" "beam-sdks-java-io-amazon-web-services" "beam-sdks-java-io-amazon-web-services2" "beam-sdks-java-io-amqp" "beam-sdks-java-io-azure" "beam-sdks-java-io-cassandra" "beam-sdks-java-io-cdap" "beam-sdks-java-io-clickhouse" "beam-sdks-java-io-common" "beam-sdks-java-io-contextualtextio" "beam-sdks-java-io-csv" "beam-sdks-java-io-debezium-expansion-service" "beam-sdks-java-io-debezium" "beam-sdks-java-io-elasticsearch" "beam-sdks-java-io-expansion-service" "beam-sdks-java-io-file-schema-transform" "beam-sdks-java-io-google-cloud-platform-expansion-service" "beam-sdks-java-io-google-cloud-platform" "beam-sdks-java-io-hadoop-common" "beam-sdks-java-io-hadoop-file-system" "beam-sdks-java-io-hadoop-format" "beam-sdks-java-io-hbase" "beam-sdks-java-io-hcatalog" "beam-sdks-java-io-influxdb" "beam-sdks-java-io-jdbc" "beam-sdks-java-io-jms" "beam-sdks-java-io-kafka" "beam-sdks-java-io-kinesis-expansion-service" "beam-sdks-java-io-kinesis" "beam-sdks-java-io-kudu" "beam-sdks-java-io-mongodb" "beam-sdks-java-io-mqtt" "beam-sdks-java-io-neo4j" "beam-sdks-java-io-parquet" "beam-sdks-java-io-pulsar" "beam-sdks-java-io-rabbitmq" "beam-sdks-java-io-redis" "beam-sdks-java-io-singlestore" "beam-sdks-java-io-snowflake-expansion-service" "beam-sdks-java-io-snowflake" "beam-sdks-java-io-solr" "beam-sdks-java-io-sparkreceiver-2" "beam-sdks-java-io-splunk" "beam-sdks-java-io-synthetic" "beam-sdks-java-io-thrift" "beam-sdks-java-io-tika" "beam-sdks-java-io-xml" "beam-sdks-java-maven-archetypes-examples" "beam-sdks-java-maven-archetypes-gcp-bom-examples" "beam-sdks-java-maven-archetypes-starter" "beam-sdks-java-nexmark" "beam-sdks-java-test-utils" "beam-sdks-java-tpcds" ) - +java_artifacts_path=$( pwd -P )/java_artifacts.txt +ARTIFACTS=( $(cat "$java_artifacts_path" |tr "\n" " ") ) echo "" echo "====================Checking Environment & Variables=================" echo "PLEASE update RC_VALIDATE_CONFIGS in file script.config first." @@ -138,7 +138,7 @@ then do echo "Artifact: ${i} - url: ${REPO_URL}/org/apache/beam/${i}/${RELEASE_VER}" done - echo "If these artifacts should no longer be published, remove them from the ARTIFACTS variable in run_rc_validation.sh and try again." + echo "If these artifacts should no longer be published, remove them from java_artifacts.txt and try again." echo "Please check any changes back in to master." exit 1 fi @@ -167,7 +167,7 @@ then do echo "Artifact: ${i} - url: ${REPO_URL}/org/apache/beam/${i}/${RELEASE_VER}" done - echo "If these artifacts should have been published (which is likely), add them from to ARTIFACTS variable in run_rc_validation.sh and try again." + echo "If these artifacts should have been published (which is likely), add them to java_artifacts.txt and try again." echo "Please check any changes back in to master." exit 1 fi From 672ba099c7d9774be35bc69332dbc4e19c5909fb Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Thu, 27 Apr 2023 09:18:46 -0400 Subject: [PATCH 3/3] use java bom contents for validation --- release/src/main/scripts/java_artifacts.txt | 120 ------------------ release/src/main/scripts/run_rc_validation.sh | 47 ++----- 2 files changed, 12 insertions(+), 155 deletions(-) delete mode 100644 release/src/main/scripts/java_artifacts.txt diff --git a/release/src/main/scripts/java_artifacts.txt b/release/src/main/scripts/java_artifacts.txt deleted file mode 100644 index 3fe333bf074d..000000000000 --- a/release/src/main/scripts/java_artifacts.txt +++ /dev/null @@ -1,120 +0,0 @@ -beam-examples-java-cdap-hubspot -beam-examples-java-cdap-salesforce -beam-examples-java-cdap-servicenow -beam-examples-java-cdap-zendesk -beam-examples-java-cdap -beam-examples-java-twitter -beam-examples-java -beam-examples-kotlin -beam-examples-multi-language -beam-model-fn-execution -beam-model-job-management -beam-model-pipeline -beam-runners-core-construction-java -beam-runners-core-java -beam-runners-direct-java -beam-runners-extensions-java-metrics -beam-runners-flink-1.12-job-server -beam-runners-flink-1.12 -beam-runners-flink-1.13-job-server -beam-runners-flink-1.13 -beam-runners-flink-1.14-job-server -beam-runners-flink-1.14 -beam-runners-flink-1.15-job-server -beam-runners-flink-1.15 -beam-runners-flink-1.16-job-server -beam-runners-flink-1.16 -beam-runners-google-cloud-dataflow-java-legacy-worker -beam-runners-google-cloud-dataflow-java -beam-runners-java-fn-execution -beam-runners-java-job-service -beam-runners-jet -beam-runners-local-java-core -beam-runners-portability-java -beam-runners-samza-job-server -beam-runners-samza -beam-runners-spark-3-job-server -beam-runners-spark-3 -beam-runners-twister2 -beam-sdks-java-bom -beam-sdks-java-core -beam-sdks-java-expansion-service-app -beam-sdks-java-expansion-service -beam-sdks-java-extensions-arrow -beam-sdks-java-extensions-avro -beam-sdks-java-extensions-euphoria -beam-sdks-java-extensions-google-cloud-platform-core -beam-sdks-java-extensions-join-library -beam-sdks-java-extensions-json-jackson -beam-sdks-java-extensions-kryo -beam-sdks-java-extensions-ml -beam-sdks-java-extensions-protobuf -beam-sdks-java-extensions-python -beam-sdks-java-extensions-sbe -beam-sdks-java-extensions-schemaio-expansion-service -beam-sdks-java-extensions-sketching -beam-sdks-java-extensions-sorter -beam-sdks-java-extensions-sql-datacatalog -beam-sdks-java-extensions-sql-expansion-service -beam-sdks-java-extensions-sql-hcatalog -beam-sdks-java-extensions-sql-jdbc -beam-sdks-java-extensions-sql-udf -beam-sdks-java-extensions-sql-zetasql -beam-sdks-java-extensions-sql -beam-sdks-java-extensions-timeseries -beam-sdks-java-extensions-zetasketch -beam-sdks-java-fn-execution -beam-sdks-java-google-cloud-platform-bom -beam-sdks-java-harness -beam-sdks-java-io-amazon-web-services -beam-sdks-java-io-amazon-web-services2 -beam-sdks-java-io-amqp -beam-sdks-java-io-azure -beam-sdks-java-io-cassandra -beam-sdks-java-io-cdap -beam-sdks-java-io-clickhouse -beam-sdks-java-io-common -beam-sdks-java-io-contextualtextio -beam-sdks-java-io-csv -beam-sdks-java-io-debezium-expansion-service -beam-sdks-java-io-debezium -beam-sdks-java-io-elasticsearch -beam-sdks-java-io-expansion-service -beam-sdks-java-io-file-schema-transform -beam-sdks-java-io-google-cloud-platform-expansion-service -beam-sdks-java-io-google-cloud-platform -beam-sdks-java-io-hadoop-common -beam-sdks-java-io-hadoop-file-system -beam-sdks-java-io-hadoop-format -beam-sdks-java-io-hbase -beam-sdks-java-io-hcatalog -beam-sdks-java-io-influxdb -beam-sdks-java-io-jdbc -beam-sdks-java-io-jms -beam-sdks-java-io-kafka -beam-sdks-java-io-kinesis-expansion-service -beam-sdks-java-io-kinesis -beam-sdks-java-io-kudu -beam-sdks-java-io-mongodb -beam-sdks-java-io-mqtt -beam-sdks-java-io-neo4j -beam-sdks-java-io-parquet -beam-sdks-java-io-pulsar -beam-sdks-java-io-rabbitmq -beam-sdks-java-io-redis -beam-sdks-java-io-singlestore -beam-sdks-java-io-snowflake-expansion-service -beam-sdks-java-io-snowflake -beam-sdks-java-io-solr -beam-sdks-java-io-sparkreceiver-2 -beam-sdks-java-io-splunk -beam-sdks-java-io-synthetic -beam-sdks-java-io-thrift -beam-sdks-java-io-tika -beam-sdks-java-io-xml -beam-sdks-java-maven-archetypes-examples -beam-sdks-java-maven-archetypes-gcp-bom-examples -beam-sdks-java-maven-archetypes-starter -beam-sdks-java-nexmark -beam-sdks-java-test-utils -beam-sdks-java-tpcds \ No newline at end of file diff --git a/release/src/main/scripts/run_rc_validation.sh b/release/src/main/scripts/run_rc_validation.sh index 39f57f5aa35e..6d9e5aa783ab 100755 --- a/release/src/main/scripts/run_rc_validation.sh +++ b/release/src/main/scripts/run_rc_validation.sh @@ -100,9 +100,6 @@ HUB_ARTIFACTS_NAME=hub-linux-amd64-${HUB_VERSION} BACKUP_BASHRC=.bashrc_backup_$(date +"%Y%m%d%H%M%S") BACKUP_M2=settings_backup_$(date +"%Y%m%d%H%M%S").xml declare -a PYTHON_VERSIONS_TO_VALIDATE=("python3.8") -# Used to define which artifacts should be present in the generated apache repository. Please add/subtract artifacts as they are published/no longer published. -java_artifacts_path=$( pwd -P )/java_artifacts.txt -ARTIFACTS=( $(cat "$java_artifacts_path" |tr "\n" " ") ) echo "" echo "====================Checking Environment & Variables=================" echo "PLEASE update RC_VALIDATE_CONFIGS in file script.config first." @@ -126,49 +123,29 @@ if [[ $confirmation != "y" ]]; then fi echo "----------------- Checking published Java artifacts (should take ~1 minute) -----------------" -FAILED=() -for i in "${ARTIFACTS[@]}" -do - curl "${REPO_URL}/org/apache/beam/${i}/${RELEASE_VER}" -f || FAILED+=($i) -done -if [ ${#FAILED[@]} != 0 ]; + +java_bom=$(curl "${REPO_URL}/org/apache/beam/beam-sdks-java-bom/${RELEASE_VER}/beam-sdks-java-bom-${RELEASE_VER}.pom") +artifacts=( $(echo $java_bom | grep -Eo "\S+?" | grep -Eo "beam-[a-zA-Z0-9.-]+") ) +if [ ${#artifacts[@]} == 0 ]; then - echo "Failed to find the following artifacts:" - for i in "${FAILED[@]}" - do - echo "Artifact: ${i} - url: ${REPO_URL}/org/apache/beam/${i}/${RELEASE_VER}" - done - echo "If these artifacts should no longer be published, remove them from java_artifacts.txt and try again." - echo "Please check any changes back in to master." + echo "Couldn't find beam-sdks-java-bom in the generated java artifact." + echo "Please check ${REPO_URL} and try regenerating the java artifacts in the build_rc step." exit 1 fi -response=$(curl "${REPO_URL}/org/apache/beam/" | grep -Eo "${REPO_URL}/\S+?\"") -published=($response) -for p in "${published[@]}" -do - in_artifacts=0 - for a in "${ARTIFACTS[@]}" - do - if [ "${REPO_URL}/org/apache/beam/${a}/\"" == $p ] - then - in_artifacts=1 - fi - done - if [ $in_artifacts == 0 ] - then - FAILED+=($a) - fi +FAILED=() +for i in "${artifacts[@]}" +do + curl "${REPO_URL}/org/apache/beam/${i}/${RELEASE_VER}" -f || FAILED+=($i) done if [ ${#FAILED[@]} != 0 ]; then - echo "The following artifacts were published but maybe should not have been:" + echo "Failed to find the following artifacts in the generated java artifact, but they were present as dependencies in beam-sdks-java-bom:" for i in "${FAILED[@]}" do echo "Artifact: ${i} - url: ${REPO_URL}/org/apache/beam/${i}/${RELEASE_VER}" done - echo "If these artifacts should have been published (which is likely), add them to java_artifacts.txt and try again." - echo "Please check any changes back in to master." + echo "Please check ${REPO_URL} and try regenerating the java artifacts in the build_rc step." exit 1 fi