From 88a1492fe707fef40598a00fed73a5a11a0f2f3b Mon Sep 17 00:00:00 2001 From: ILYA Khlopotov Date: Tue, 10 Mar 2015 15:31:23 -0700 Subject: [PATCH 1/4] Intoroduce `couch_define_db` behaviour COUCHDB-2635 --- src/couch.app.src | 5 +- src/couch_app.erl | 1 + src/couch_auth_cache.erl | 16 +++--- src/couch_db.erl | 7 +++ src/couch_dbs.erl | 23 ++++++++ src/couch_define_db.erl | 11 ++++ src/couch_server.erl | 3 +- src/couch_system_dbs.erl | 117 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 173 insertions(+), 10 deletions(-) create mode 100644 src/couch_dbs.erl create mode 100644 src/couch_define_db.erl create mode 100644 src/couch_system_dbs.erl diff --git a/src/couch.app.src b/src/couch.app.src index 54eedf8d..ec569d4c 100644 --- a/src/couch.app.src +++ b/src/couch.app.src @@ -23,7 +23,10 @@ couch_secondary_services, couch_server, couch_sup, - couch_task_status + couch_task_status, + couch_dbs, + couch_define_db, + couch_system_dbs ]}, {mod, {couch_app, []}}, {applications, [ diff --git a/src/couch_app.erl b/src/couch_app.erl index d284c2bf..feceb6c4 100644 --- a/src/couch_app.erl +++ b/src/couch_app.erl @@ -19,6 +19,7 @@ -export([start/2, stop/1]). start(_Type, _) -> + couch_system_dbs:subscribe(), case couch_sup:start_link() of {ok, _} = Resp -> Resp; diff --git a/src/couch_auth_cache.erl b/src/couch_auth_cache.erl index 0b154899..856a4e84 100644 --- a/src/couch_auth_cache.erl +++ b/src/couch_auth_cache.erl @@ -69,7 +69,7 @@ get_user_creds(_Req, UserName) -> validate_user_creds(UserCreds). update_user_creds(_Req, UserDoc, _AuthCtx) -> - DbNameList = config:get("couch_httpd_auth", "authentication_db", "_users"), + DbNameList = couch_dbs:name("authentication_db"), couch_util:with_db(?l2b(DbNameList), fun(UserDb) -> {ok, _NewRev} = couch_db:update_doc(UserDb, UserDoc, []), ok @@ -148,7 +148,7 @@ init(_) -> ?STATE = ets:new(?STATE, [set, protected, named_table]), ?BY_USER = ets:new(?BY_USER, [set, protected, named_table]), ?BY_ATIME = ets:new(?BY_ATIME, [ordered_set, private, named_table]), - AuthDbName = config:get("couch_httpd_auth", "authentication_db"), + AuthDbName = couch_dbs:name("authentication_db"), process_flag(trap_exit, true), ok = config:listen_for_changes(?MODULE, nil), {ok, Listener} = couch_event:link_listener( @@ -270,7 +270,7 @@ clear_cache(State) -> reinit_cache(State) -> NewState = clear_cache(State), - AuthDbName = ?l2b(config:get("couch_httpd_auth", "authentication_db")), + AuthDbName = ?l2b(couch_dbs:name("authentication_db")), true = ets:insert(?STATE, {auth_db_name, AuthDbName}), AuthDb = open_auth_db(), true = ets:insert(?STATE, {auth_db, AuthDb}), @@ -409,7 +409,7 @@ exec_if_auth_db(Fun, DefRes) -> open_auth_db() -> [{auth_db_name, DbName}] = ets:lookup(?STATE, auth_db_name), - {ok, AuthDb} = ensure_users_db_exists(DbName, [sys_db]), + {ok, AuthDb} = ensure_users_db_exists(DbName), AuthDb. @@ -430,14 +430,14 @@ get_user_props_from_db(UserName) -> nil ). -ensure_users_db_exists(DbName, Options) -> - Options1 = [?ADMIN_CTX, nologifmissing | Options], - case couch_db:open(DbName, Options1) of +ensure_users_db_exists(DbName) -> + Options = [?ADMIN_CTX|couch_dbs:options(DbName)], + case couch_db:open(DbName, Options) of {ok, Db} -> ensure_auth_ddoc_exists(Db, <<"_design/_auth">>), {ok, Db}; _Error -> - {ok, Db} = couch_db:create(DbName, Options1), + {ok, Db} = couch_db:create(DbName, Options), ok = ensure_auth_ddoc_exists(Db, <<"_design/_auth">>), {ok, Db} end. diff --git a/src/couch_db.erl b/src/couch_db.erl index 0d4857f5..05af553a 100644 --- a/src/couch_db.erl +++ b/src/couch_db.erl @@ -32,6 +32,7 @@ -export([reopen/1, is_system_db/1, compression/1, make_doc/5]). -export([load_validation_funs/1]). -export([check_md5/2, with_stream/3]). +-export([normalize_dbname/1]). -include_lib("couch/include/couch_db.hrl"). @@ -1464,3 +1465,9 @@ select_gt(V1, _V2) -> V1. select_lt(V1, V2) when V1 > V2 -> V2; select_lt(V1, _V2) -> V1. + +normalize_dbname(<<"shards/", _/binary>> = Path) -> + lists:last(binary:split(mem3:dbname(Path), <<"/">>, [global])); +normalize_dbname(DbName) -> + DbName. + diff --git a/src/couch_dbs.erl b/src/couch_dbs.erl new file mode 100644 index 00000000..03f35c3e --- /dev/null +++ b/src/couch_dbs.erl @@ -0,0 +1,23 @@ +-module(couch_dbs). + +-behaviour(couch_define_db). + +-export([databases/0, validate_name/1, options/1, name/1]). + +databases() -> + [name("authentication_db")]. + +validate_name(DbName) -> + name("authentication_db") == couch_db:normalize_dbname(DbName). + +options(_Name) -> + [ + {before_doc_update, fun couch_users_db:before_doc_update/2}, + {after_doc_read, fun couch_users_db:after_doc_read/2}, + sys_db, + nologifmissing, + local + ]. + +name("authentication_db") -> + config:get("couch_httpd_auth", "authentication_db", "_users"). diff --git a/src/couch_define_db.erl b/src/couch_define_db.erl new file mode 100644 index 00000000..8824a7b7 --- /dev/null +++ b/src/couch_define_db.erl @@ -0,0 +1,11 @@ +-module(couch_define_db). + +-export([behaviour_info/1]). + +behaviour_info(callbacks) -> + [ + {databases, 0}, + {validate_name, 1}, + {options, 1}, + {name, 1} + ]. diff --git a/src/couch_server.erl b/src/couch_server.erl index 228edc30..a9091ae1 100644 --- a/src/couch_server.erl +++ b/src/couch_server.erl @@ -148,7 +148,8 @@ path_ends_with(Path, Suffix) -> check_dbname(#server{dbname_regexp=RegExp}, DbName) -> case re:run(DbName, RegExp, [{capture, none}]) of nomatch -> - case lists:member(DbName, ?SYSTEM_DATABASES) of + Name = couch_db:normalize_dbname(DbName), + case couch_system_dbs:is_system_db(Name) of true -> ok; false -> {error, illegal_database_name, DbName} end; diff --git a/src/couch_system_dbs.erl b/src/couch_system_dbs.erl new file mode 100644 index 00000000..03a85503 --- /dev/null +++ b/src/couch_system_dbs.erl @@ -0,0 +1,117 @@ +-module(couch_system_dbs). + +-behaviour(config_listener). +-vsn(1). + +-export([subscribe/0, before_doc_update/1, after_doc_update/1]). +-export([system_dbs/0, local_dbs/0, options/1, is_system_db/1]). + +% config_listener api +-export([handle_config_change/5, handle_config_terminate/3]). + +subscribe() -> + ok = config:listen_for_changes(?MODULE, nil), + register_all(). + +before_doc_update(Name) -> + Callbacks = get_env(system_dbs_before_doc_update, []), + proplists:get_value(Name, Callbacks, undefined). + +after_doc_update(Name) -> + Callbacks = get_env(system_dbs_after_doc_update, []), + proplists:get_value(Name, Callbacks, undefined). + +system_dbs() -> + get_env(system_dbs_all, []). + +local_dbs() -> + get_env(system_dbs_local, []). + +options(Name) -> + Options = get_env(system_dbs_options, []), + proplists:get_value(Name, Options, []). + +is_system_db(Name) -> + lists:keymember(Name, 1, get_env(system_dbs_all, [])). + +handle_config_change("couchdb", "db_definitions", _, _, _) -> + register_all(), + {ok, nil}; +handle_config_change(_, _, _, _, _) -> + {ok, nil}. + +handle_config_terminate(_, stop, _) -> ok; +handle_config_terminate(_, _, _) -> + spawn(fun() -> + timer:sleep(5000), + config:listen_for_changes(?MODULE, nil) + end). + +%% private functions + +register_all() -> + Definitions = definitions(), + register_system_dbs(Definitions), + register_local_dbs(Definitions), + register_sys_callbacks(Definitions), + register_options(Definitions). + +definitions() -> + ConfigStr = config:get("couchdb", "db_definitions", "[]"), + {ok, Modules} = couch_util:parse_term(ConfigStr), + lists:flatten([define(Module) || Module <- Modules]). + +get_env(Key, Default) -> + case application:get_env(couch, Key) of + undefined -> + Default; + {ok, Value} -> + Value + end. + +define(Module) -> + [{Id, Module} || Id <- Module:databases()]. + +register_system_dbs(Definitions) -> + application:set_env(couch, system_dbs_all, Definitions). + +register_local_dbs(Definitions) -> + application:set_env(couch, system_dbs_local, local_dbs(Definitions)). + +register_sys_callbacks(Definitions) -> + {Before, After} = callbacks(Definitions), + application:set_env(couch, system_dbs_before_doc_update, Before), + application:set_env(couch, system_dbs_after_doc_update, After). + +register_options(Definitions) -> + application:set_env(couch, system_dbs_options, options_int(Definitions)). + +local_dbs(Definitions) -> + lists:filtermap(fun({Name, Module}) -> + case lists:member(local, Module:options(Name)) of + true -> + {true, list_to_binary(Name)}; + false -> + false + end + end, Definitions). + +callbacks(Definitions) -> + lists:foldl(fun({Id, Options}, {BeforeAcc0, AfterAcc0}) -> + BeforeAcc = + append_if_callback_set(before_doc_update, Id, Options, BeforeAcc0), + AfterAcc = + append_if_callback_set(before_doc_update, Id, Options, AfterAcc0), + {BeforeAcc, AfterAcc} + end, {[], []}, options_int(Definitions)). + +options_int(Definitions) -> + [{Id, Module:options(Id)} || {Id, Module} <- Definitions]. + +append_if_callback_set(Type, Id, Options, Acc) -> + case proplists:get_value(Type, Options) of + undefined -> + Acc; + Fun -> + [{Id, Fun}|Acc] + end. From 96ba6c9a844785e4f8705dff4e663e51935140b7 Mon Sep 17 00:00:00 2001 From: ILYA Khlopotov Date: Tue, 10 Mar 2015 15:33:01 -0700 Subject: [PATCH 2/4] Use options defined in `couch_dbs` for system_dbs COUCHDB-2635 --- src/couch_server.erl | 45 ++++++++++---------------------------------- 1 file changed, 10 insertions(+), 35 deletions(-) diff --git a/src/couch_server.erl b/src/couch_server.erl index a9091ae1..0869e80a 100644 --- a/src/couch_server.erl +++ b/src/couch_server.erl @@ -76,8 +76,7 @@ sup_start_link() -> gen_server:start_link({local, couch_server}, couch_server, [], []). -open(DbName, Options0) -> - Options = maybe_add_sys_db_callbacks(DbName, Options0), +open(DbName, Options) -> Ctx = couch_util:get_value(user_ctx, Options, #user_ctx{}), case ets:lookup(couch_dbs, DbName) of [#db{fd=Fd, fd_monitor=Lock} = Db] when Lock =/= locked -> @@ -103,8 +102,7 @@ update_lru(DbName, Options) -> close_lru() -> gen_server:call(couch_server, close_lru). -create(DbName, Options0) -> - Options = maybe_add_sys_db_callbacks(DbName, Options0), +create(DbName, Options) -> case gen_server:call(couch_server, {create, DbName, Options}, infinity) of {ok, #db{fd=Fd} = Db} -> Ctx = couch_util:get_value(user_ctx, Options, #user_ctx{}), @@ -116,35 +114,6 @@ create(DbName, Options0) -> delete(DbName, Options) -> gen_server:call(couch_server, {delete, DbName, Options}, infinity). -maybe_add_sys_db_callbacks(DbName, Options) when is_binary(DbName) -> - maybe_add_sys_db_callbacks(?b2l(DbName), Options); -maybe_add_sys_db_callbacks(DbName, Options) -> - DbsDbName = config:get("mem3", "shards_db", "_dbs"), - NodesDbName = config:get("mem3", "nodes_db", "_nodes"), - IsReplicatorDb = DbName == config:get("replicator", "db", "_replicator") orelse - path_ends_with(DbName, <<"_replicator">>), - IsUsersDb = DbName ==config:get("couch_httpd_auth", "authentication_db", "_users") orelse - path_ends_with(DbName, <<"_users">>), - if - DbName == DbsDbName -> - [sys_db | Options]; - DbName == NodesDbName -> - [sys_db | Options]; - IsReplicatorDb -> - [{before_doc_update, fun couch_replicator_manager:before_doc_update/2}, - {after_doc_read, fun couch_replicator_manager:after_doc_read/2}, - sys_db | Options]; - IsUsersDb -> - [{before_doc_update, fun couch_users_db:before_doc_update/2}, - {after_doc_read, fun couch_users_db:after_doc_read/2}, - sys_db | Options]; - true -> - Options - end. - -path_ends_with(Path, Suffix) -> - Suffix == lists:last(binary:split(mem3:dbname(Path), <<"/">>, [global])). - check_dbname(#server{dbname_regexp=RegExp}, DbName) -> case re:run(DbName, RegExp, [{capture, none}]) of nomatch -> @@ -392,7 +361,8 @@ handle_call({open_result, DbName, Error}, {FromPid, _Tag}, Server) -> Server end, {reply, ok, db_closed(NewServer, Db#db.options)}; -handle_call({open, DbName, Options}, From, Server) -> +handle_call({open, DbName, Options0}, From, Server) -> + Options = maybe_add_sys_db_options(DbName, Options0), case ets:lookup(couch_dbs, DbName) of [] -> DbNameList = binary_to_list(DbName), @@ -419,7 +389,8 @@ handle_call({open, DbName, Options}, From, Server) -> [#db{} = Db] -> {reply, {ok, Db}, Server} end; -handle_call({create, DbName, Options}, From, Server) -> +handle_call({create, DbName, Options0}, From, Server) -> + Options = maybe_add_sys_db_options(DbName, Options0), DbNameList = binary_to_list(DbName), Filepath = get_full_filename(Server, DbNameList), case check_dbname(Server, DbNameList) of @@ -557,3 +528,7 @@ db_closed(Server, Options) -> false -> Server#server{dbs_open=Server#server.dbs_open - 1}; true -> Server end. + +maybe_add_sys_db_options(DbName, Options) -> + Name = couch_db:normalize_dbname(DbName), + couch_system_dbs:options(Name) ++ Options. From b19cc956d454056271e9f0847f29e0dab868818f Mon Sep 17 00:00:00 2001 From: ILYA Khlopotov Date: Tue, 10 Mar 2015 15:34:16 -0700 Subject: [PATCH 3/4] Add couch_db:before_doc_updates for fabric COUCHDB-2635 --- src/couch_db.erl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/couch_db.erl b/src/couch_db.erl index 05af553a..1dc0561a 100644 --- a/src/couch_db.erl +++ b/src/couch_db.erl @@ -33,6 +33,7 @@ -export([load_validation_funs/1]). -export([check_md5/2, with_stream/3]). -export([normalize_dbname/1]). +-export([before_doc_update/2]). -include_lib("couch/include/couch_db.hrl"). @@ -1471,3 +1472,10 @@ normalize_dbname(<<"shards/", _/binary>> = Path) -> normalize_dbname(DbName) -> DbName. +before_doc_update(DbName, Docs) -> + case couch_system_dbs:before_doc_update(normalize_dbname(DbName)) of + undefined -> + Docs; + Fun -> + lists:map(Fun, Docs) + end. From 5ec649fccf7d5ce42a6ad41673d335f788fcf76b Mon Sep 17 00:00:00 2001 From: ILYA Khlopotov Date: Tue, 10 Mar 2015 15:35:01 -0700 Subject: [PATCH 4/4] Add couch_users_db:should_strip_public_fields Move the check from couch_mrview_http to couch_users_db. COUCHDB-2635 --- src/couch_users_db.erl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/couch_users_db.erl b/src/couch_users_db.erl index 3b768623..266526de 100644 --- a/src/couch_users_db.erl +++ b/src/couch_users_db.erl @@ -13,6 +13,7 @@ -module(couch_users_db). -export([before_doc_update/2, after_doc_read/2, strip_non_public_fields/1]). +-export([should_strip_public_fields/0]). -include_lib("couch/include/couch_db.hrl"). @@ -119,3 +120,13 @@ strip_non_public_fields(#doc{body={Props}}=Doc) -> Public = re:split(config:get("couch_httpd_auth", "public_fields", ""), "\\s*,\\s*", [{return, binary}]), Doc#doc{body={[{K, V} || {K, V} <- Props, lists:member(K, Public)]}}. + +should_strip_public_fields() -> + UsersDbPublic = config:get("couch_httpd_auth", "users_db_public", "false"), + PublicFields = config:get("couch_httpd_auth", "public_fields"), + case {UsersDbPublic, PublicFields} of + {"true", PublicFields} when PublicFields =/= undefined -> + true; + _ -> + false + end.