[DO NOT SUBMIT][TESTING] testing changes to RC validation

gradle.properties

@@ -24,8 +24,8 @@ offlineRepositoryRoot=offline-repository
 signing.gnupg.executable=gpg
 signing.gnupg.useLegacyGpg=true
 
-version=2.37.0-SNAPSHOT
-sdk_version=2.37.0.dev
+version=2.37.0
+sdk_version=2.37.0
 
 javaVersion=1.8
 

release/src/main/python-release/python_release_automation_utils.sh

@@ -228,7 +228,7 @@ function cleanup_pubsub() {
 #   $2 - pid: the pid of running pipeline
 #   $3 - running_job (DataflowRunner only): the job id of streaming pipeline running on DataflowRunner
 #######################################
-function verify_steaming_result() {
+function verify_streaming_result() {
   retry=3
   should_see="Python: "
   while(( $retry > 0 )); do
@@ -295,9 +295,11 @@ function verify_user_score() {
 function verify_hourly_team_score() {
   retry=3
   should_see='AntiqueBrassPlatypus'
+  runner=$1
+
   while(( $retry >= 0 )); do
     if [[ $retry > 0 ]]; then
-      bq_pull_result=$(bq head -n 500 $DATASET.hourly_team_score_python_$1)
+      bq_pull_result=$(bq head -n 500 ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner})
       if [[ $bq_pull_result = *"$should_see"* ]]; then
         echo "SUCCEED: hourly_team_score example successful run on $1-runner"
         break
@@ -307,14 +309,38 @@ function verify_hourly_team_score() {
         sleep 15
       fi
     else
-      echo "FAILED: HourlyTeamScore example failed running on $1-runner. \
-        Did not found scores of team $should_see in $DATASET.leader_board"
+      echo "FAILED: HourlyTeamScore example failed running on $runner runner. \
+        Did not found scores of team $should_see in ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}"
       complete "FAILED"
       exit 1
     fi
   done
 }
 
+function cleanup_hourly_team_score() {
+  retry=3
+  runner=$1
+
+  echo "Removing previously created table ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}..."
+  bq rm -q -f -t "${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}"
+
+  while(( $retry >= 0 )); do
+    if [[ $retry > 0 ]]; then
+      bq_ls_result=$(bq ls $DATASET)
+      if [[ $bq_ls_result = *"${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}"* ]]; then
+        retry=$(($retry-1))
+        echo "${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner} not cleaned up yet, waiting"
+        sleep 1000
+      else
+        echo "Confirmed ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner} removed before running new test."
+        break
+      fi
+    else
+      echo "WARNING: Unable to clean up table ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}. \
+            You may need to manually run 'bq rm -r -t ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}'."
+    fi
+  done
+}
 
 # Python RC configurations
 VERSION=$(get_version)
@@ -336,4 +362,5 @@ PUBSUB_SUBSCRIPTION='wordstream-python-sub2'
 # Mobile Gaming Configurations
 DATASET='beam_postrelease_mobile_gaming'
 USERSCORE_OUTPUT_PREFIX='python-userscore_result'
+HOURLY_TEAM_SCORE_TABLE_PREFIX='hourly_team_score_python'
 GAME_INPUT_DATA='gs://dataflow-samples/game/5000_gaming_data.csv'

release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh

@@ -100,12 +100,15 @@ function verify_userscore_dataflow() {
 #######################################
 function verify_hourlyteamscore_direct() {
   print_separator "Running HourlyTeamScore example with DirectRunner"
+  # Clean up old bq tables
+  cleanup_hourly_team_score "direct"
+
   python -m apache_beam.examples.complete.game.hourly_team_score \
     --project=$PROJECT_ID \
     --dataset=$DATASET \
     --input=$GAME_INPUT_DATA \
     --temp_location=gs://$BUCKET_NAME/temp/ \
-    --table="hourly_team_score_python_direct"
+    --table="${HOURLY_TEAM_SCORE_TABLE_PREFIX}_direct"
 
   verify_hourly_team_score "direct"
 }
@@ -121,6 +124,9 @@ function verify_hourlyteamscore_direct() {
 #######################################
 function verify_hourlyteamscore_dataflow() {
   print_separator "Running HourlyTeamScore example with DataflowRunner"
+  # Clean up old bq tables
+  cleanup_hourly_team_score "dataflow"
+
   python -m apache_beam.examples.complete.game.hourly_team_score \
     --project=$PROJECT_ID \
     --region=$REGION_ID \
@@ -129,7 +135,7 @@ function verify_hourlyteamscore_dataflow() {
     --temp_location=gs://$BUCKET_NAME/temp/ \
     --sdk_location $BEAM_PYTHON_SDK \
     --input=$GAME_INPUT_DATA \
-    --table="hourly_team_score_python_dataflow"
+    --table="${HOURLY_TEAM_SCORE_TABLE_PREFIX}_dataflow"
 
   verify_hourly_team_score "dataflow"
 }

release/src/main/python-release/run_release_candidate_python_quickstart.sh

@@ -155,7 +155,7 @@ function verify_streaming_wordcount_direct() {
 
   # verify result
   run_pubsub_publish
-  verify_steaming_result "DirectRunner" $pid
+  verify_streaming_result "DirectRunner" $pid
 
   kill -9 $pid
   sleep 10
@@ -194,7 +194,7 @@ function verify_streaming_wordcount_dataflow() {
   # verify result
   run_pubsub_publish
   sleep 420
-  verify_steaming_result "DataflowRunner" $pid $running_job
+  verify_streaming_result "DataflowRunner" $pid $running_job
 
   kill -9 $pid
   gcloud dataflow jobs cancel $running_job

runners/google-cloud-dataflow-java/build.gradle

@@ -54,8 +54,8 @@ processResources {
   filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [
     'dataflow.legacy_environment_major_version' : '8',
     'dataflow.fnapi_environment_major_version' : '8',
-    'dataflow.legacy_container_version' : 'beam-2.37.0',
-    'dataflow.fnapi_container_version' : 'beam-2.37.0',
+    'dataflow.legacy_container_version' : '2.37.0',
+    'dataflow.fnapi_container_version' : '2.37.0',
     'dataflow.container_base_repository' : 'gcr.io/cloud-dataflow/v1beta3',
   ]
 }

sdks/go/pkg/beam/core/core.go

@@ -27,5 +27,5 @@ const (
 	// SdkName is the human readable name of the SDK for UserAgents.
 	SdkName = "Apache Beam SDK for Go"
 	// SdkVersion is the current version of the SDK.
-	SdkVersion = "2.37.0.dev"
+	SdkVersion = "2.37.0"
 )

sdks/python/apache_beam/version.py

@@ -17,4 +17,4 @@
 
 """Apache Beam SDK version information and utilities."""
 
-__version__ = '2.37.0.dev'
+__version__ = '2.37.0'