Add .retriable feature to Http

lib/http.rb

@@ -6,6 +6,7 @@
 require "http/timeout/global"
 require "http/chainable"
 require "http/client"
+require "http/retriable/client"
 require "http/connection"
 require "http/options"
 require "http/feature"

lib/http/chainable.rb

@@ -246,6 +246,28 @@ def use(*features)
       branch default_options.with_features(features)
     end
 
+    # Returns retriable client instance, which retries requests if they failed
+    # due to some socket errors or response status is `5xx`.
+    #
+    # @example Usage
+    #
+    #   # Retry max 5 times with randomly growing delay between retries
+    #   HTTP.retriable.get(url)
+    #
+    #   # Retry max 3 times with randomly growing delay between retries
+    #   HTTP.retriable(:times => 3).get(url)
+    #
+    #   # Retry max 3 times with 1 sec delay between retries
+    #   HTTP.retriable(:times => 3, :delay => proc { 1 }).get(url)
+    #
+    #   # Retry max 3 times with geometrically progressed delay between retries
+    #   HTTP.retriable(:times => 3, :delay => proc { |i| 1 + i*i }).get(url)
+    #
+    # @param (see Performer#initialize)
+    def retriable(**options)
+      Retriable::Client.new(Retriable::Performer.new(options), default_options)
+    end
+
     private
 
     # :nodoc:

lib/http/retriable/client.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "http/retriable/performer"
+
+module HTTP
+  module Retriable
+    # Retriable version of HTTP::Client.
+    #
+    # @see http://www.rubydoc.info/gems/http/HTTP/Client
+    class Client < HTTP::Client
+      # @param [Performer] performer
+      # @param [HTTP::Options, Hash] options
+      def initialize(performer, options)
+        @performer = performer
+        super(options)
+      end
+
+      # Overriden version of `HTTP::Client#make_request`.
+      #
+      # Monitors request/response phase with performer.
+      #
+      # @see http://www.rubydoc.info/gems/http/HTTP/Client:perform
+      def perform(req, options)
+        @performer.perform(self, req) { super(req, options) }
+      end
+
+      private
+
+      # Overriden version of `HTTP::Chainable#branch`.
+      #
+      # @return [HTTP::Retriable::Client]
+      def branch(options)
+        Retriable::Client.new(@performer, options)
+      end
+    end
+  end
+end

lib/http/retriable/delay_calculator.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module HTTP
+  module Retriable
+    # @api private
+    class DelayCalculator
+      def initialize(opts)
+        @max_delay = opts.fetch(:max_delay, Float::MAX).to_f
+        if (delay = opts[:delay]).respond_to?(:call)
+          @delay_proc = opts.fetch(:delay)
+        else
+          @delay = delay
+        end
+      end
+
+      def call(iteration, response)
+        delay = if response && (retry_header = response.headers["Retry-After"])
+                  delay_from_retry_header(retry_header)
+                else
+                  calculate_delay_from_iteration(iteration)
+                end
+
+        ensure_dealy_in_bounds(delay)
+      end
+
+      RFC2822_DATE_REGEX = /^
+        (?:Sun|Mon|Tue|Wed|Thu|Fri|Sat),\s+
+        (?:0[1-9]|[1-2]?[0-9]|3[01])\s+
+        (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+
+        (?:19[0-9]{2}|[2-9][0-9]{3})\s+
+        (?:2[0-3]|[0-1][0-9]):(?:[0-5][0-9]):(?:60|[0-5][0-9])\s+
+        GMT
+      $/x.freeze
+
+      # Spec for Retry-After header
+      # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+      def delay_from_retry_header(value)
+        value = value.to_s.strip
+
+        case value
+        when RFC2822_DATE_REGEX then DateTime.rfc2822(value).to_time - Time.now.utc
+        when /^\d+$/            then value.to_i
+        else 0
+        end
+      end
+
+      def calculate_delay_from_iteration(iteration)
+        if @delay_proc
+          @delay_proc.call(iteration)
+        elsif @delay
+          @delay
+        else
+          delay = 2**(iteration - 1) - 1
+          delay_noise = rand
+          delay + delay_noise
+        end
+      end
+
+      def ensure_dealy_in_bounds(delay)
+        [0, [delay, @max_delay].min].max
+      end
+    end
+  end
+end

lib/http/retriable/errors.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module HTTP
+  # Retriable performance ran out of attempts
+  class OutOfRetriesError < Error
+    attr_accessor :response
+
+    def cause=(exception)
+      @cause = exception
+    end
+
+    def cause
+      @cause || super
+    end
+  end
+end

lib/http/retriable/performer.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require "date"
+require "http"
+require "http/retriable/errors"
+require "http/retriable/delay_calculator"
+require "openssl"
+
+module HTTP
+  module Retriable
+    # Request performing watchdog.
+    # @api private
+    class Performer
+      # Exceptions we should retry
+      RETRIABLE_ERRORS = [
+        HTTP::TimeoutError,
+        HTTP::ConnectionError,
+        IO::EAGAINWaitReadable,
+        Errno::ECONNRESET,
+        Errno::ECONNREFUSED,
+        Errno::EHOSTUNREACH,
+        OpenSSL::SSL::SSLError,
+        EOFError,
+        IOError
+      ].freeze
+
+      # @param [Hash] opts
+      # @option opts [#to_i] :tries (5)
+      # @option opts [#call, #to_i] :delay (DELAY_PROC)
+      # @option opts [Array(Exception)] :exceptions (RETRIABLE_ERRORS)
+      # @option opts [Array(#to_i)] :retry_statuses
+      # @option opts [#call] :on_retry
+      # @option opts [#to_f] :max_delay (Float::MAX)
+      # @option opts [#call] :should_retry
+      def initialize(opts)
+        @exception_classes = opts.fetch(:exceptions, RETRIABLE_ERRORS)
+        @retry_statuses = opts[:retry_statuses]
+        @tries = opts.fetch(:tries, 5).to_i
+        @on_retry = opts.fetch(:on_retry, ->(*) {})
+        @should_retry_proc = opts[:should_retry]
+        @delay_calculator = DelayCalculator.new(opts)
+      end
+
+      # Watches request/response execution.
+      #
+      # If any of {RETRIABLE_ERRORS} occur or response status is `5xx`, retries
+      # up to `:tries` amount of times. Sleeps for amount of seconds calculated
+      # with `:delay` proc before each retry.
+      #
+      # @see #initialize
+      # @api private
+      def perform(client, req)
+        1.upto(Float::INFINITY) do |attempt| # infinite loop with index
+          err, res = try_request { yield }
+
+          if retry_request?(req, err, res, attempt)
+            begin
+              wait_for_retry_or_raise(req, err, res, attempt)
+            ensure
+              # Some servers support Keep-Alive on any response. Thus we should
+              # flush response before retry, to avoid state error (when socket
+              # has pending response data and we try to write new request).
+              # Alternatively, as we don't need response body here at all, we
+              # are going to close client, effectivle closing underlying socket
+              # and resetting client's state.
+              client.close
+            end
+          elsif err
+            client.close
+            raise err
+          elsif res
+            return res
+          end
+        end
+      end
+
+      def calculate_delay(iteration, response)
+        @delay_calculator.call(iteration, response)
+      end
+
+      private
+
+      # rubocop:disable Lint/RescueException
+      def try_request
+        err, res = nil
+
+        begin
+          res = yield
+        rescue Exception => e
+          err = e
+        end
+
+        [err, res]
+      end
+      # rubocop:enable Lint/RescueException
+
+      def retry_request?(req, err, res, attempt)
+        if @should_retry_proc
+          @should_retry_proc.call(req, err, res, attempt)
+        elsif err
+          retry_exception?(err)
+        else
+          retry_response?(res)
+        end
+      end
+
+      def retry_exception?(err)
+        @exception_classes.any? { |e| err.is_a?(e) }
+      end
+
+      def retry_response?(res)
+        return false unless @retry_statuses
+
+        response_status = res.status.to_i
+        retry_matchers = [@retry_statuses].flatten
+
+        retry_matchers.any? do |matcher|
+          case matcher
+          when Range then matcher.cover?(response_status)
+          when Numeric then matcher == response_status
+          else matcher.call(response_status)
+          end
+        end
+      end
+
+      def wait_for_retry_or_raise(req, err, res, attempt)
+        if attempt < @tries
+          @on_retry.call(req, err, res)
+          sleep calculate_delay(attempt, res)
+        else
+          res&.flush
+          raise out_of_retries_error(req, res, err)
+        end
+      end
+
+      # Builds OutOfRetriesError
+      #
+      # @param request [HTTP::Request]
+      # @param status [HTTP::Response, nil]
+      # @param exception [Exception, nil]
+      def out_of_retries_error(request, response, exception)
+        message = "#{request.verb.to_s.upcase} <#{request.uri}> failed"
+
+        message += " with #{response.status}" if response
+        message += ":#{exception}" if exception
+
+        HTTP::OutOfRetriesError.new(message).tap do |ex|
+          ex.cause = exception
+          ex.response = response
+        end
+      end
+    end
+  end
+end

spec/lib/http/retriable/delay_calculator_spec.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+RSpec.describe HTTP::Retriable::DelayCalculator do
+  let(:response) do
+    HTTP::Response.new(
+      :status  => 200,
+      :version => "1.1",
+      :headers => {},
+      :body    => "Hello world!",
+      :uri     => "http://example.com/",
+      :request => request
+    )
+  end
+
+  let(:request) do
+    HTTP::Request.new(
+      :verb => :get,
+      :uri  => "http://example.com"
+    )
+  end
+
+  def call_delay(iterations, **options)
+    described_class.new(options).call(iterations, response)
+  end
+
+  def call_retry_header(value, **options)
+    response.headers["Retry-After"] = value
+    described_class.new(options).call(rand(1...100), response)
+  end
+
+  it "prevents negative sleep time" do
+    expect(call_delay(20, :delay => -20)).to eq 0
+  end
+
+  it "backs off exponentially" do
+    expect(call_delay(1)).to be_between 0, 1
+    expect(call_delay(2)).to be_between 1, 2
+    expect(call_delay(3)).to be_between 3, 4
+    expect(call_delay(4)).to be_between 7, 8
+    expect(call_delay(5)).to be_between 15, 16
+  end
+
+  it "can have a maximum wait time" do
+    expect(call_delay(1, :max_delay => 5)).to be_between 0, 1
+    expect(call_delay(5, :max_delay => 5)).to eq 5
+  end
+
+  it "respects Retry-After headers as integer" do
+    delay_time = rand(6...2500)
+    header_value = delay_time.to_s
+    expect(call_retry_header(header_value)).to eq delay_time
+    expect(call_retry_header(header_value, :max_delay => 5)).to eq 5
+  end
+
+  it "respects Retry-After headers as rfc2822 timestamp" do
+    delay_time = rand(6...2500)
+    header_value = (Time.now.gmtime + delay_time).to_datetime.rfc2822.sub("+0000", "GMT")
+    expect(call_retry_header(header_value)).to be_within(1).of(delay_time)
+    expect(call_retry_header(header_value, :max_delay => 5)).to eq 5
+  end
+
+  it "respects Retry-After headers as rfc2822 timestamp in the past" do
+    delay_time = rand(6...2500)
+    header_value = (Time.now.gmtime - delay_time).to_datetime.rfc2822.sub("+0000", "GMT")
+    expect(call_retry_header(header_value)).to eq 0
+  end
+
+  it "does not error on invalid Retry-After header" do
+    [ # invalid strings
+      "This is a string with a number 5 in it",
+      "8 Eight is the first digit in this string",
+      "This is a string with a #{Time.now.gmtime.to_datetime.rfc2822} timestamp in it"
+    ].each do |header_value|
+      expect(call_retry_header(header_value)).to eq 0
+    end
+  end
+end