From dc996b10a34dec8b242fc23d728f6bc89938a4bc Mon Sep 17 00:00:00 2001 From: Garren Smith Date: Tue, 27 Nov 2018 16:40:02 +0200 Subject: [PATCH 1/2] Add Global timeout for map/reduce queries This makes the global timeout for a map/reduce and all_docs request configurable via the config. It separates the config into global queries and partition queries so that it is possible to make the global timeout is for partitioned queries --- src/fabric/src/fabric_util.erl | 9 ++++- src/fabric/src/fabric_view_all_docs.erl | 2 +- src/fabric/src/fabric_view_map.erl | 2 +- src/fabric/src/fabric_view_reduce.erl | 2 +- test/elixir/test/partition_all_docs_test.exs | 35 ++++++++++++++++++++ 5 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl index d65d3c81d13..1ba1d17ea79 100644 --- a/src/fabric/src/fabric_util.erl +++ b/src/fabric/src/fabric_util.erl @@ -15,7 +15,7 @@ -export([submit_jobs/3, submit_jobs/4, cleanup/1, recv/4, get_db/1, get_db/2, error_info/1, update_counter/3, remove_ancestors/2, create_monitors/1, kv/2, remove_down_workers/2, doc_id_and_rev/1]). --export([request_timeout/0, attachments_timeout/0, all_docs_timeout/0]). +-export([request_timeout/0, attachments_timeout/0, all_docs_timeout/0, view_timeout/1]). -export([log_timeout/2, remove_done_workers/2]). -export([is_users_db/1, is_replicator_db/1]). -export([open_cluster_db/1, open_cluster_db/2]). @@ -65,6 +65,13 @@ all_docs_timeout() -> attachments_timeout() -> timeout("attachments", "600000"). +view_timeout(Args) -> + PartitionQuery = couch_mrview_util:get_extra(Args, partition, false), + case PartitionQuery of + false -> timeout("view", "infinity"); + _ -> timeout("partition_view", "infinity") + end. + timeout(Type, Default) -> case config:get("fabric", Type ++ "_timeout", Default) of "infinity" -> infinity; diff --git a/src/fabric/src/fabric_view_all_docs.erl b/src/fabric/src/fabric_view_all_docs.erl index 4b412a6838d..9049eaa90dd 100644 --- a/src/fabric/src/fabric_view_all_docs.erl +++ b/src/fabric/src/fabric_view_all_docs.erl @@ -127,7 +127,7 @@ go(DbName, _Options, Workers, QueryArgs, Callback, Acc0) -> update_seq = case UpdateSeq of true -> []; false -> nil end }, case rexi_utils:recv(Workers, #shard.ref, fun handle_message/3, - State, infinity, 5000) of + State, fabric_util:view_timeout(QueryArgs), 5000) of {ok, NewState} -> {ok, NewState#collector.user_acc}; {timeout, NewState} -> diff --git a/src/fabric/src/fabric_view_map.erl b/src/fabric/src/fabric_view_map.erl index b3d768a5138..1bbc6d20103 100644 --- a/src/fabric/src/fabric_view_map.erl +++ b/src/fabric/src/fabric_view_map.erl @@ -81,7 +81,7 @@ go(DbName, Workers, {map, View, _}, Args, Callback, Acc0) -> update_seq = case UpdateSeq of true -> []; false -> nil end }, case rexi_utils:recv(Workers, #shard.ref, fun handle_message/3, - State, infinity, 1000 * 60 * 60) of + State, fabric_util:view_timeout(Args), 1000 * 60 * 60) of {ok, NewState} -> {ok, NewState#collector.user_acc}; {timeout, NewState} -> diff --git a/src/fabric/src/fabric_view_reduce.erl b/src/fabric/src/fabric_view_reduce.erl index f52061a4c3f..b157c37e5f1 100644 --- a/src/fabric/src/fabric_view_reduce.erl +++ b/src/fabric/src/fabric_view_reduce.erl @@ -87,7 +87,7 @@ go2(DbName, Workers, {red, {_, Lang, View}, _}=VInfo, Args, Callback, Acc0) -> update_seq = case UpdateSeq of true -> []; false -> nil end }, try rexi_utils:recv(Workers, #shard.ref, fun handle_message/3, - State, infinity, 1000 * 60 * 60) of + State, fabric_util:view_timeout(Args), 1000 * 60 * 60) of {ok, NewState} -> {ok, NewState#collector.user_acc}; {timeout, NewState} -> diff --git a/test/elixir/test/partition_all_docs_test.exs b/test/elixir/test/partition_all_docs_test.exs index 0941daf59f5..308c1b4393e 100644 --- a/test/elixir/test/partition_all_docs_test.exs +++ b/test/elixir/test/partition_all_docs_test.exs @@ -115,4 +115,39 @@ defmodule PartitionAllDocsTest do assert length(ids) == 1 assert ids == ["foo:22"] end + + # This test is timing based so it could be a little flaky. + # If that turns out to be the case we should probably just skip it + test "partition _all_docs with timeout", context do + on_exit(fn -> + resp = Couch.get("/_membership") + %{:body => body} = resp + + Enum.each(body["all_nodes"], fn node -> + resp = Couch.put("/_node/#{node}/_config/fabric/partition_view_timeout", body: "\"3600000\"") + assert resp.status_code == 200 + end) + end) + + resp = Couch.get("/_membership") + %{:body => body} = resp + + Enum.each(body["all_nodes"], fn node -> + resp = Couch.put("/_node/#{node}/_config/fabric/partition_view_timeout", body: "\"1\"") + assert resp.status_code == 200 + end) + + db_name = context[:db_name] + create_partition_docs(db_name) + + retry_until(fn -> + url = "/#{db_name}/_partition/foo/_all_docs" + case Couch.get(url) do + %{:body => %{"reason" => reason}} -> + Regex.match?(~r/not be processed in a reasonable amount of time./, reason) + _ -> + false + end + end) + end end From dadc3ffe63ac93b2fa93e1fe14a2023b21efbc20 Mon Sep 17 00:00:00 2001 From: Garren Smith Date: Thu, 29 Nov 2018 08:57:50 +0200 Subject: [PATCH 2/2] add default fabric request timeouts --- rel/overlay/etc/default.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index ae9d3133e65..a32cfdf12ed 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -194,6 +194,11 @@ port = 6984 ; changes_duration = ; shard_timeout_factor = 2 ; uuid_prefix_len = 7 +; request_timeout = 60000 +; all_docs_timeout = 10000 +; attachments_timeout = 60000 +; view_timeout = 3600000 +; partition_view_timeout = 3600000 ; [rexi] ; buffer_count = 2000