diff --git a/gradle.properties b/gradle.properties index 5876c34c349a..02e0608839a6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -24,8 +24,8 @@ offlineRepositoryRoot=offline-repository signing.gnupg.executable=gpg signing.gnupg.useLegacyGpg=true -version=2.37.0-SNAPSHOT -sdk_version=2.37.0.dev +version=2.37.0 +sdk_version=2.37.0 javaVersion=1.8 diff --git a/release/src/main/python-release/python_release_automation_utils.sh b/release/src/main/python-release/python_release_automation_utils.sh index 5d0e1c6eff76..b328c428a797 100644 --- a/release/src/main/python-release/python_release_automation_utils.sh +++ b/release/src/main/python-release/python_release_automation_utils.sh @@ -228,7 +228,7 @@ function cleanup_pubsub() { # $2 - pid: the pid of running pipeline # $3 - running_job (DataflowRunner only): the job id of streaming pipeline running on DataflowRunner ####################################### -function verify_steaming_result() { +function verify_streaming_result() { retry=3 should_see="Python: " while(( $retry > 0 )); do @@ -295,9 +295,11 @@ function verify_user_score() { function verify_hourly_team_score() { retry=3 should_see='AntiqueBrassPlatypus' + runner=$1 + while(( $retry >= 0 )); do if [[ $retry > 0 ]]; then - bq_pull_result=$(bq head -n 500 $DATASET.hourly_team_score_python_$1) + bq_pull_result=$(bq head -n 500 ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}) if [[ $bq_pull_result = *"$should_see"* ]]; then echo "SUCCEED: hourly_team_score example successful run on $1-runner" break @@ -307,14 +309,38 @@ function verify_hourly_team_score() { sleep 15 fi else - echo "FAILED: HourlyTeamScore example failed running on $1-runner. \ - Did not found scores of team $should_see in $DATASET.leader_board" + echo "FAILED: HourlyTeamScore example failed running on $runner runner. \ + Did not found scores of team $should_see in ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}" complete "FAILED" exit 1 fi done } +function cleanup_hourly_team_score() { + retry=3 + runner=$1 + + echo "Removing previously created table ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}..." + bq rm -q -f -t "${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}" + + while(( $retry >= 0 )); do + if [[ $retry > 0 ]]; then + bq_ls_result=$(bq ls $DATASET) + if [[ $bq_ls_result = *"${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}"* ]]; then + retry=$(($retry-1)) + echo "${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner} not cleaned up yet, waiting" + sleep 1000 + else + echo "Confirmed ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner} removed before running new test." + break + fi + else + echo "WARNING: Unable to clean up table ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}. \ + You may need to manually run 'bq rm -r -t ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}'." + fi + done +} # Python RC configurations VERSION=$(get_version) @@ -336,4 +362,5 @@ PUBSUB_SUBSCRIPTION='wordstream-python-sub2' # Mobile Gaming Configurations DATASET='beam_postrelease_mobile_gaming' USERSCORE_OUTPUT_PREFIX='python-userscore_result' +HOURLY_TEAM_SCORE_TABLE_PREFIX='hourly_team_score_python' GAME_INPUT_DATA='gs://dataflow-samples/game/5000_gaming_data.csv' diff --git a/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh b/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh index 30d8cea2bd95..efd903da48b8 100755 --- a/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh +++ b/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh @@ -100,12 +100,15 @@ function verify_userscore_dataflow() { ####################################### function verify_hourlyteamscore_direct() { print_separator "Running HourlyTeamScore example with DirectRunner" + # Clean up old bq tables + cleanup_hourly_team_score "direct" + python -m apache_beam.examples.complete.game.hourly_team_score \ --project=$PROJECT_ID \ --dataset=$DATASET \ --input=$GAME_INPUT_DATA \ --temp_location=gs://$BUCKET_NAME/temp/ \ - --table="hourly_team_score_python_direct" + --table="${HOURLY_TEAM_SCORE_TABLE_PREFIX}_direct" verify_hourly_team_score "direct" } @@ -121,6 +124,9 @@ function verify_hourlyteamscore_direct() { ####################################### function verify_hourlyteamscore_dataflow() { print_separator "Running HourlyTeamScore example with DataflowRunner" + # Clean up old bq tables + cleanup_hourly_team_score "dataflow" + python -m apache_beam.examples.complete.game.hourly_team_score \ --project=$PROJECT_ID \ --region=$REGION_ID \ @@ -129,7 +135,7 @@ function verify_hourlyteamscore_dataflow() { --temp_location=gs://$BUCKET_NAME/temp/ \ --sdk_location $BEAM_PYTHON_SDK \ --input=$GAME_INPUT_DATA \ - --table="hourly_team_score_python_dataflow" + --table="${HOURLY_TEAM_SCORE_TABLE_PREFIX}_dataflow" verify_hourly_team_score "dataflow" } diff --git a/release/src/main/python-release/run_release_candidate_python_quickstart.sh b/release/src/main/python-release/run_release_candidate_python_quickstart.sh index 6f1e10da33b3..3af527acfa2e 100755 --- a/release/src/main/python-release/run_release_candidate_python_quickstart.sh +++ b/release/src/main/python-release/run_release_candidate_python_quickstart.sh @@ -155,7 +155,7 @@ function verify_streaming_wordcount_direct() { # verify result run_pubsub_publish - verify_steaming_result "DirectRunner" $pid + verify_streaming_result "DirectRunner" $pid kill -9 $pid sleep 10 @@ -194,7 +194,7 @@ function verify_streaming_wordcount_dataflow() { # verify result run_pubsub_publish sleep 420 - verify_steaming_result "DataflowRunner" $pid $running_job + verify_streaming_result "DataflowRunner" $pid $running_job kill -9 $pid gcloud dataflow jobs cancel $running_job diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 832c7059d2cd..edabb8b17f15 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -54,8 +54,8 @@ processResources { filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [ 'dataflow.legacy_environment_major_version' : '8', 'dataflow.fnapi_environment_major_version' : '8', - 'dataflow.legacy_container_version' : 'beam-2.37.0', - 'dataflow.fnapi_container_version' : 'beam-2.37.0', + 'dataflow.legacy_container_version' : '2.37.0', + 'dataflow.fnapi_container_version' : '2.37.0', 'dataflow.container_base_repository' : 'gcr.io/cloud-dataflow/v1beta3', ] } diff --git a/sdks/go/pkg/beam/core/core.go b/sdks/go/pkg/beam/core/core.go index 0f23e864052f..0c959e822943 100644 --- a/sdks/go/pkg/beam/core/core.go +++ b/sdks/go/pkg/beam/core/core.go @@ -27,5 +27,5 @@ const ( // SdkName is the human readable name of the SDK for UserAgents. SdkName = "Apache Beam SDK for Go" // SdkVersion is the current version of the SDK. - SdkVersion = "2.37.0.dev" + SdkVersion = "2.37.0" ) diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py index f80b33928688..944583dab0d1 100644 --- a/sdks/python/apache_beam/version.py +++ b/sdks/python/apache_beam/version.py @@ -17,4 +17,4 @@ """Apache Beam SDK version information and utilities.""" -__version__ = '2.37.0.dev' +__version__ = '2.37.0'