From 8fa9163786940162183d33f39ab94301ab470493 Mon Sep 17 00:00:00 2001 From: Oguz Kocer Date: Wed, 10 May 2023 15:45:33 -0400 Subject: [PATCH 1/4] Adds the option to auto-retry for downloading metadata from GlotPress --- .../actions/common/gp_downloadmetadata_action.rb | 9 ++++++++- .../helper/metadata_download_helper.rb | 13 ++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb b/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb index acc002a56..3e05c3821 100644 --- a/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb +++ b/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb @@ -10,12 +10,13 @@ def self.run(params) UI.message "Locales: #{params[:locales].inspect}" UI.message "Source locale: #{params[:source_locale].nil? ? '-' : params[:source_locale]}" UI.message "Path: #{params[:download_path]}" + UI.message "Auto-retry: #{params[:auto_retry]}" # Check download path FileUtils.mkdir_p(params[:download_path]) # Download - downloader = Fastlane::Helper::MetadataDownloader.new(params[:download_path], params[:target_files]) + downloader = Fastlane::Helper::MetadataDownloader.new(params[:download_path], params[:target_files], params[:auto_retry]) params[:locales].each do |loc| if loc.is_a?(Array) @@ -68,6 +69,12 @@ def self.available_options env_name: 'FL_DOWNLOAD_METADATA_DOWNLOAD_PATH', description: 'The path of the target files', type: String), + FastlaneCore::ConfigItem.new(key: :auto_retry, + env_name: 'FL_DOWNLOAD_METADATA_AUTO_RETRY', + description: 'Whether to auto retry downloads after Too Many Requests error', + type: FastlaneCore::Boolean, + optional: true, + default_value: false), ] end diff --git a/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb b/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb index a31add3aa..61488abe4 100644 --- a/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb +++ b/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb @@ -4,11 +4,14 @@ module Fastlane module Helper class MetadataDownloader + AUTO_RETRY_SLEEP_TIME = 20 + attr_reader :target_folder, :target_files - def initialize(target_folder, target_files) + def initialize(target_folder, target_files, auto_retry) @target_folder = target_folder @target_files = target_files + @auto_retry = auto_retry @alternates = {} end @@ -112,8 +115,12 @@ def handle_glotpress_download(response:, locale:, is_source:) UI.message("Received 301 for `#{locale}`. Following redirect...") download(locale, response.header['location'], is_source) when '429' - # We got rate-limited, offer to try again - if UI.confirm("Retry downloading `#{locale}` after receiving 429 from the API?") + # We got rate-limited, auto_retry or offer to try again with a prompt + if @auto_retry + UI.message("Received 429 for `#{locale}`. Auto retrying in #{AUTO_RETRY_SLEEP_TIME} seconds...") + sleep(AUTO_RETRY_SLEEP_TIME) + download(locale, response.uri, is_source) + elsif UI.confirm("Retry downloading `#{locale}` after receiving 429 from the API?") download(locale, response.uri, is_source) else UI.error("Abandoning `#{locale}` download as requested.") From 17ca0b7e8915cad2b545ce7237d1e9ff762df5f5 Mon Sep 17 00:00:00 2001 From: Oguz Kocer Date: Thu, 18 May 2023 05:53:18 -0400 Subject: [PATCH 2/4] Limit number of auto_retry attempts in metadata_download_helper --- .../wpmreleasetoolkit/helper/metadata_download_helper.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb b/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb index 61488abe4..bdae83c68 100644 --- a/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb +++ b/lib/fastlane/plugin/wpmreleasetoolkit/helper/metadata_download_helper.rb @@ -5,6 +5,7 @@ module Fastlane module Helper class MetadataDownloader AUTO_RETRY_SLEEP_TIME = 20 + MAX_AUTO_RETRY_ATTEMPTS = 30 attr_reader :target_folder, :target_files @@ -13,6 +14,7 @@ def initialize(target_folder, target_files, auto_retry) @target_files = target_files @auto_retry = auto_retry @alternates = {} + @auto_retry_attempt_counter = 0 end # Downloads data from GlotPress, in JSON format @@ -116,9 +118,10 @@ def handle_glotpress_download(response:, locale:, is_source:) download(locale, response.header['location'], is_source) when '429' # We got rate-limited, auto_retry or offer to try again with a prompt - if @auto_retry + if @auto_retry && @auto_retry_attempt_counter <= MAX_AUTO_RETRY_ATTEMPTS UI.message("Received 429 for `#{locale}`. Auto retrying in #{AUTO_RETRY_SLEEP_TIME} seconds...") sleep(AUTO_RETRY_SLEEP_TIME) + @auto_retry_attempt_counter += 1 download(locale, response.uri, is_source) elsif UI.confirm("Retry downloading `#{locale}` after receiving 429 from the API?") download(locale, response.uri, is_source) From 5c19fca498f52e34c8c6e177b7afdf63e66b5dde Mon Sep 17 00:00:00 2001 From: Oguz Kocer Date: Thu, 18 May 2023 06:00:10 -0400 Subject: [PATCH 3/4] Adds CHANGELOG entry for new auto_retry option in gp_downloadmetadata_action --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c012fa712..cde10c5c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ _None_ ### New Features -_None_ +- Adds auto_retry option to `gp_downloadmetadata_action`. [#474] ### Bug Fixes From dc56ef0ca29e7977db8b29b358f9d352d052b00f Mon Sep 17 00:00:00 2001 From: Oguz Kocer Date: Fri, 19 May 2023 00:26:09 -0400 Subject: [PATCH 4/4] Change auto_retry gp_downloadmetadata_action parameter to true --- .../actions/common/gp_downloadmetadata_action.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb b/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb index 3e05c3821..2de44300b 100644 --- a/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb +++ b/lib/fastlane/plugin/wpmreleasetoolkit/actions/common/gp_downloadmetadata_action.rb @@ -74,7 +74,7 @@ def self.available_options description: 'Whether to auto retry downloads after Too Many Requests error', type: FastlaneCore::Boolean, optional: true, - default_value: false), + default_value: true), ] end