From 7e75e0437347c307f295a007265cae5fffc39bdb Mon Sep 17 00:00:00 2001 From: n1mus <709030+n1mus@users.noreply.github.com> Date: Tue, 8 Feb 2022 04:51:07 +0000 Subject: [PATCH 1/2] recursive obj subsetting --- relation_engine_server/main.py | 4 +- relation_engine_server/utils/config.py | 1 - relation_engine_server/utils/ensure_specs.py | 21 +++- relation_engine_server/utils/wait_for.py | 1 - spec/test/test_ensure_specs.py | 116 +++++++++++++++++++ 5 files changed, 138 insertions(+), 5 deletions(-) diff --git a/relation_engine_server/main.py b/relation_engine_server/main.py index e359dc6b..1a924577 100644 --- a/relation_engine_server/main.py +++ b/relation_engine_server/main.py @@ -43,8 +43,8 @@ def return_error(error_dict, code): @app.route("/", methods=["GET"]) def root(): """Server status.""" - if os.path.exists(".git/refs/heads/master"): - with open(".git/refs/heads/master", "r") as fd: + if os.path.exists(".git/refs/heads/develop"): + with open(".git/refs/heads/develop", "r") as fd: commit_hash = fd.read().strip() else: commit_hash = "unknown" diff --git a/relation_engine_server/utils/config.py b/relation_engine_server/utils/config.py index a548fe39..fdea1b05 100644 --- a/relation_engine_server/utils/config.py +++ b/relation_engine_server/utils/config.py @@ -31,7 +31,6 @@ def get_config(): db_readonly_user = os.environ.get("DB_READONLY_USER", db_user) db_readonly_pass = os.environ.get("DB_READONLY_PASS", db_pass) api_url = db_url + "/_db/" + db_name + "/_api" - print(db_user, db_pass) return { "auth_url": auth_url, "workspace_url": workspace_url, diff --git a/relation_engine_server/utils/ensure_specs.py b/relation_engine_server/utils/ensure_specs.py index 584d5cdc..835deb06 100644 --- a/relation_engine_server/utils/ensure_specs.py +++ b/relation_engine_server/utils/ensure_specs.py @@ -12,7 +12,7 @@ def match(spec_local, specs_server): for spec_server in specs_server: - if spec_local.items() <= spec_server.items(): + if is_obj_subset_rec(spec_local, spec_server): return True return False @@ -228,6 +228,25 @@ def excise_namespace(analyzer_name: str) -> str: return analyzer_name.split("::")[-1] +def is_obj_subset_rec( + l: Union[dict, list, float, str, int], + r: Union[dict, list, float, str, int], +): + """ + Compare two JSON objects, to see if, essentially, l <= r + If comparing dicts, recursively compare + If comparing lists, shallowly compare. For now, YAGN more + """ + if isinstance(l, dict) and isinstance(r, dict): + return all( + [k in r.keys() and is_obj_subset_rec(l[k], r[k]) for k in l.keys()] + ) # ignore: typing + elif isinstance(l, list) and isinstance(r, list): + return all([le in r for le in l]) + else: + return l == r # noqa: E741 + + def mod_obj_literal( spec_unit: Union[list, dict], literal_type: type, diff --git a/relation_engine_server/utils/wait_for.py b/relation_engine_server/utils/wait_for.py index 4aed73c2..06b4a59a 100644 --- a/relation_engine_server/utils/wait_for.py +++ b/relation_engine_server/utils/wait_for.py @@ -41,7 +41,6 @@ def wait_for_service(service_list: List[str]) -> None: try: conf = service_conf[name] auth = (_CONF["db_user"], _CONF["db_pass"]) - print("auth is", auth) resp = requests.get(conf["url"], auth=auth) if conf.get("raise_for_status"): resp.raise_for_status() diff --git a/spec/test/test_ensure_specs.py b/spec/test/test_ensure_specs.py index a6e85bbe..3f95cad9 100644 --- a/spec/test/test_ensure_specs.py +++ b/spec/test/test_ensure_specs.py @@ -1,6 +1,7 @@ import unittest from unittest import mock import copy +import json from relation_engine_server.utils import arango_client from relation_engine_server.utils.ensure_specs import ( @@ -11,6 +12,7 @@ ensure_views, ensure_analyzers, ensure_all, + is_obj_subset_rec, mod_obj_literal, round_float, excise_namespace, @@ -130,6 +132,120 @@ def test_ensure_all__fail__mock_arango_client_get_all_things(self): # --- Unit tests --- # ------------------ + def test_is_obj_subset_rec(self): + """ + For comparing JSON objects + Roughly check l <= r, with recursive checks done with dicts + """ + exp_pass = [ + ({"hi": 1}, {"hi": 1}), + ({"hi": 1}, {"hi": 1, "hello": 2}), + ({}, {}), + ({}, {"hi": 1}), + ( + {"hi": 1, "hello": {"cat": 3, "sat": 2}}, + { + "hi": 1, + "hello": {"cat": 3, "sat": 2, "hat": 3, "bat": {}, "map": []}, + "hey": 5, + "aloha": [{}], + }, + ), + ] + exp_fail = [ + ({"hi": 1}, {}), + ({"hi": {}}, {}), + ({"hi": 1}, {"hi": {}}), + ({"hi": 1, "hello": 2}, {"hi": 1}), + ( + {"hi": 1, "hello": {"cat": 3, "sat": 2, "hat": 3}}, + {"hi": 1, "hello": {"cat": 3, "sat": 2}}, + ), + ( + {"hi": 1, "hello": {"cat": 3, "sat": {}}}, + {"hi": 1, "hello": {"cat": 3, "sat": 2}}, + ), + ( + {"hi": 1, "hello": {"cat": 3}, "hey": 5}, + {"hi": 1, "hello": {"cat": 3, "sat": 2}}, + ), + ( + { + "hi": 1, + "hello": {"cat": 3, "sat": 2, "hat": 3}, + "hey": 5, + "howdy": 6, + }, + {"hi": 1, "hello": {"cat": 3, "sat": 2}}, + ), + ] + + for loc, srv in exp_pass: + self.assertTrue(is_obj_subset_rec(loc, srv)) + for loc, srv in exp_fail: + self.assertFalse(is_obj_subset_rec(loc, srv)) + + def test_is_obj_subset_rec__Reactions(self): + """ + Test the recursive subset functions using Reactions.json view spec + """ + # Local spec + local = [view for view in get_local_views()[1] if view["name"] == "Reactions"][ + 0 + ] + # Server spec + # From Aardvark, but with "name" key/field added + # as seems to happen with GET + server = json.loads( + """ +{ + "name": "Reactions", + "writebufferIdle": 64, + "writebufferActive": 0, + "type": "arangosearch", + "primarySort": [], + "writebufferSizeMax": 33554432, + "commitIntervalMsec": 1000, + "consolidationPolicy": { + "type": "bytes_accum", + "threshold": 0.10000000149011612 + }, + "globallyUniqueId": "h5455DEB9D2A1/9853332", + "cleanupIntervalStep": 10, + "id": "9853332", + "links": { + "rxn_reaction": { + "analyzers": [ + "identity" + ], + "fields": { + "name": { + "analyzers": [ + "text_en" + ] + }, + "aliases": { + "analyzers": [ + "text_en" + ] + }, + "id": { + "analyzers": [ + "text_en" + ] + } + }, + "includeAllFields": true, + "storeValues": "none", + "trackListPositions": false + } + }, + "consolidationIntervalMsec": 60000 +}""" + ) + mod_obj_literal(server, float, round_float) + self.assertTrue(is_obj_subset_rec(local, server)) + def _copy_mod_obj_literal(self, obj, literal_type, func): obj = copy.deepcopy(obj) mod_obj_literal(obj, literal_type, func) From 2e406f0386d900f0efb4de28f1d15866df1e8f7e Mon Sep 17 00:00:00 2001 From: n1mus <709030+n1mus@users.noreply.github.com> Date: Tue, 8 Feb 2022 17:52:06 +0000 Subject: [PATCH 2/2] doc func --- relation_engine_server/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relation_engine_server/main.py b/relation_engine_server/main.py index 1a924577..e2f14a5f 100644 --- a/relation_engine_server/main.py +++ b/relation_engine_server/main.py @@ -42,7 +42,7 @@ def return_error(error_dict, code): @app.route("/", methods=["GET"]) def root(): - """Server status.""" + """Server status. develop is default branch""" if os.path.exists(".git/refs/heads/develop"): with open(".git/refs/heads/develop", "r") as fd: commit_hash = fd.read().strip()