From 1b12a573586adfa88354a8ae2808ef0bcf2fbdb8 Mon Sep 17 00:00:00 2001 From: Lorenzo Date: Sun, 5 Aug 2018 18:15:40 +0100 Subject: [PATCH 1/9] Update README.md --- README.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c6ceca8..f421dd5 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,22 @@ # python-hydra-agent -The python-hydra-agent is a smart python hydra client which is working with [hydrus](https://github.com/HTTP-APIs/hydrus). +For a general introduction to Hydra Ecosystem, see [hydraecosystem.org](http://hydraecosystem.org). -It caches the server data from hydra server for fast data querying. +`python-hydra-agent` is a smart Hydra client implemented in Python which works with [hydrus](https://github.com/HTTP-APIs/hydrus). Reference implementation is [Heracles.ts](https://github.com/HydraCG/Heracles.ts). Smart clients are generic automated clients that establish resilient connected data networks leveraging knowledge graphs. -It uses Redis to cache the data at the end of the client. +## General characteristics -So, Data is loaded from the server and store in Redis memory as a graph using redisgraph. +The client is designed to: +* Cache metadata from the Hydra server it connects to, to allow querying on the client-side; +* Use Redis as a graph-store leveraging `redisgraph` (see [here](https://oss.redislabs.com/redisgraph/)); +* simply, metadata and data are loaded from the server and stored in Redis; +* The graph can be queried using OpenCypher. -With the help of Redis, clients become faster and easier to query the data. +The starting objective is to create a querying layer that is able to reach data in one or more Hydra srever/s. Leveraging Redis, clients can construct their own representation of the data stored in one or more Hydra servers; querying the data as they need it, and respond complex semantic queries. This will allow any client connected to any server to have access to an "aggregated view" of the connected network (the network of all the servers it connects to). + +## Missing bits at the moment +* For now it is a proof-of-concept, only `GET` functionality +* Soon to develop, a reliable synchronization mechanism to allow strong consistency between server-side data and client-side representation. ## Installation From 46ea2132ca9fb5e4b22a1ae76d38c593c47f3db4 Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Thu, 20 Dec 2018 10:56:28 +0530 Subject: [PATCH 2/9] Create .travis.yml --- .travis.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..4dd5337 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,16 @@ +language: python +services: + - docker + +before_script: + - docker run -d -p 6379:6379 -it --rm --name redisgraph redislabs/redisgraph + +python: + - "3.5" + - "3.5-dev" # 3.5 development branch + - "3.6" + - "3.6-dev" # 3.6 development branch + - "3.7-dev" +install: + - pip install -e . +script: python -m unittest discover From f0459056c18a56d5341caa25e62e87b2c624b21a Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Tue, 1 Jan 2019 17:24:11 +0530 Subject: [PATCH 3/9] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4dd5337..4922154 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,5 +12,5 @@ python: - "3.6-dev" # 3.6 development branch - "3.7-dev" install: - - pip install -e . + - pip install . script: python -m unittest discover From 38aec3d76399dd2f26b8b1a8cc511c626d3a3711 Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Tue, 1 Jan 2019 17:33:34 +0530 Subject: [PATCH 4/9] Update .travis.yml --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4922154..cafaac6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,5 +12,7 @@ python: - "3.6-dev" # 3.6 development branch - "3.7-dev" install: + - pip install -e git+https://github.com/HTTP-APIs/hydrus#egg=hydrus - pip install . + script: python -m unittest discover From d496724485996e77090bddfae8297a20d7281e22 Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Tue, 1 Jan 2019 17:42:15 +0530 Subject: [PATCH 5/9] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cafaac6..4da93f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,6 @@ python: - "3.7-dev" install: - pip install -e git+https://github.com/HTTP-APIs/hydrus#egg=hydrus - - pip install . + - pip install -e . script: python -m unittest discover From 7168dbea2d5bbd5d5f351d92aec6dd5b707d2604 Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Tue, 1 Jan 2019 17:48:07 +0530 Subject: [PATCH 6/9] Rename tests to make discoverable with `python -m unittest discover` --- hydra_redis/tests/__init__.py | 0 hydra_redis/tests/{querying_test.py => test_querying.py} | 0 hydra_redis/tests/{redis_test.py => test_redis.py} | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 hydra_redis/tests/__init__.py rename hydra_redis/tests/{querying_test.py => test_querying.py} (100%) rename hydra_redis/tests/{redis_test.py => test_redis.py} (100%) diff --git a/hydra_redis/tests/__init__.py b/hydra_redis/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hydra_redis/tests/querying_test.py b/hydra_redis/tests/test_querying.py similarity index 100% rename from hydra_redis/tests/querying_test.py rename to hydra_redis/tests/test_querying.py diff --git a/hydra_redis/tests/redis_test.py b/hydra_redis/tests/test_redis.py similarity index 100% rename from hydra_redis/tests/redis_test.py rename to hydra_redis/tests/test_redis.py From 5ba1b6a4132fab4bd92f57ce30ea1df347c99744 Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Tue, 1 Jan 2019 17:50:38 +0530 Subject: [PATCH 7/9] Add Travis build status to README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9be243c..1197cde 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# python-hydra-agent +# python-hydra-agent [![Build Status](https://travis-ci.com/HTTP-APIs/python-hydra-agent.svg?branch=master)](https://travis-ci.com/HTTP-APIs/python-hydra-agent) For a general introduction to Hydra Ecosystem, see [hydraecosystem.org](http://hydraecosystem.org). From 13ff1c2bf2e9d79dc5b59cef2d0108442106a4f6 Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Tue, 1 Jan 2019 18:23:06 +0530 Subject: [PATCH 8/9] Fix test discover issue after merging --- hydra_agent/tests/__init__.py | 0 hydra_agent/tests/{querying_test.py => test_querying.py} | 0 hydra_agent/tests/{redis_test.py => test_redis.py} | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 hydra_agent/tests/__init__.py rename hydra_agent/tests/{querying_test.py => test_querying.py} (100%) rename hydra_agent/tests/{redis_test.py => test_redis.py} (100%) diff --git a/hydra_agent/tests/__init__.py b/hydra_agent/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hydra_agent/tests/querying_test.py b/hydra_agent/tests/test_querying.py similarity index 100% rename from hydra_agent/tests/querying_test.py rename to hydra_agent/tests/test_querying.py diff --git a/hydra_agent/tests/redis_test.py b/hydra_agent/tests/test_redis.py similarity index 100% rename from hydra_agent/tests/redis_test.py rename to hydra_agent/tests/test_redis.py From 13c24e6e6b5ac68dc194d94daec9d130f382d73b Mon Sep 17 00:00:00 2001 From: Akshay Dahiya Date: Tue, 1 Jan 2019 18:24:27 +0530 Subject: [PATCH 9/9] remove old hydra_redis dir --- hydra_redis/tests/__init__.py | 0 hydra_redis/tests/test_querying.py | 107 --------------------------- hydra_redis/tests/test_redis.py | 114 ----------------------------- 3 files changed, 221 deletions(-) delete mode 100644 hydra_redis/tests/__init__.py delete mode 100644 hydra_redis/tests/test_querying.py delete mode 100644 hydra_redis/tests/test_redis.py diff --git a/hydra_redis/tests/__init__.py b/hydra_redis/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/hydra_redis/tests/test_querying.py b/hydra_redis/tests/test_querying.py deleted file mode 100644 index cb3bd90..0000000 --- a/hydra_redis/tests/test_querying.py +++ /dev/null @@ -1,107 +0,0 @@ -import unittest -import urllib.request -import json -import redis -from hydrus.hydraspec import doc_maker -from os import sys, path -from hydra_redis import querying_mechanism - -class TestQueryingMechanism(unittest.TestCase): - - def setUp(self): - url = "https://storage.googleapis.com/api4/api" - vocab_url = url + "/" + "vocab" - response = urllib.request.urlopen(vocab_url) - apidoc = json.loads(response.read().decode('utf-8')) - api_doc = doc_maker.create_doc(apidoc) - self.query_facades = querying_mechanism.QueryFacades(api_doc, url, True) - self.query_facades.initialize(True) - self.test_database = redis.StrictRedis(host='localhost', port=6379, db=5) - - def test_1_classendpoint(self): - """Test for class endpoint""" - check_data = [['p.properties', 'p.id', 'p.type'], - ["['Location']",'vocab:EntryPoint/Location','Location']] - query = "show classEndpoints" - data = self.query_facades.user_query(query) - for check in check_data: - flag = False - if check[0] in str(data) and check[1] in str(data) and check[2] in str(data): - flag = True - if flag: - self.assertTrue(True) - else: - self.assertTrue(False) - - def test_2_collectionendpoint(self): - """Test for collection endpoint""" - check_data = ["ControllerLogCollection", - "DroneLogCollection", - "AnomalyCollection", - "DroneCollection", - "CommandCollection", - "HttpApiLogCollection", - "DatastreamCollection", - "MessageCollection"] - query = "show collectionEndpoints" - data = self.query_facades.user_query(query) - for check in check_data: - if check not in str(data): - self.assertTrue(False) - self.assertTrue(True) - - def test_3_CommandCollectionmember(self): - """ - Test for all Commands in CommandCollection. - Data is already stored in check_data from the static data url. - Check_data is used for compare the data retrieve by querying process. - """ - check_data = ['[]'] - query = "show CommandCollection members" - data = self.query_facades.user_query(query) - self.assertEqual(data[1],check_data) - - def test_4_ControllerLogCollectionmember(self): - """ - Test for all controller logs for ControllerLogCollection. - Whole object of ControllerLogCollection is stored in check data. - Check_data is used for compare the data retrieve by querying process. - """ - check_data = [{'@id': '/api/ControllerLogCollection/65', - '@type': 'ControllerLog'}, - {'@id': '/api/ControllerLogCollection/183', - '@type': 'ControllerLog'}, - {'@id': '/api/ControllerLogCollection/374', - '@type': 'ControllerLog'}] - query = "show ControllerLogCollection members" - data = self.query_facades.user_query(query) - # Make data searchable and comaprable. - data1 = str(data[1]).replace('"', '') - # data retrive from the memory can be distributed: - # like type can be at first position and id can be at second. - # So, check_data split in 3 parts. - # And check all parts are in data retrieve. - if str( - check_data[0]) in data1 and str( - check_data[1]) in data1 and str( - check_data[2]) in data1: - self.assertTrue(True) - else: - self.assertTrue(False) - - - def test_5_DatastreamCollectionmember(self): - """Test for all datastream with Drone ID 2""" - check_data = ['/api/DatastreamCollection/19'] - query = "show DatastreamCollection members" - data = self.query_facades.user_query(query) - # Here are find the datastream only for those which have DroneID 2. - query = "show DroneID 2 and type Datastream" - data = self.query_facades.user_query(query) - self.assertEqual(data,check_data) - - def tearDown(self): - self.test_database.flushdb() - -if __name__ == "__main__": - unittest.main() diff --git a/hydra_redis/tests/test_redis.py b/hydra_redis/tests/test_redis.py deleted file mode 100644 index 4a9ae38..0000000 --- a/hydra_redis/tests/test_redis.py +++ /dev/null @@ -1,114 +0,0 @@ -import unittest -import redis - - -class Tests: - def entry_point(self): - """Test for testing the data stored in entrypoint endpoint""" - - print("entrypoint db=0") - r = redis.StrictRedis(host='localhost', port=6379, db=0) - reply = r.execute_command('GRAPH.QUERY', - 'apidoc', "MATCH (p:id) RETURN p") - property_list = [] - flag = 0 - for objects in reply: - for obj in objects: - if flag == 0: - string = obj.decode('utf-8') - map_string = map(str.strip, string.split(',')) - property_list = list(map_string) - check = property_list.pop() - property_list.append(check.replace("\x00", "")) - print(property_list) - flag += 1 - break - if ("p.id" in property_list and - "p.url" in property_list and - "p.supportedOperation" in property_list): - return True - else: - return False - - def collection_endpoints(self): - """Test for testing the data stored in collection endpoints""" - - print("collection endpoints db=0") - r = redis.StrictRedis(host='localhost', port=6379, db=0) - reply = r.execute_command('GRAPH.QUERY', - 'apidoc', "MATCH (p:collection) RETURN p") - property_list = [] - flag = 0 - for objects in reply: - for obj in objects: - if flag == 0: - string = obj.decode('utf-8') - map_string = map(str.strip, string.split(',')) - property_list = list(map_string) - check = property_list.pop() - property_list.append(check.replace("\x00", "")) - print(property_list) - flag += 1 - break - if ("p.id" in property_list and - "p.operations" in property_list and - "p.members" in property_list): - return True - else: - return False - - def class_endpoints(self): - """Test for testing the data stored in classes endpoints""" - - print("class endpoints db=0") - r = redis.StrictRedis(host='localhost', port=6379, db=0) - reply = r.execute_command('GRAPH.QUERY', - 'apidoc', "MATCH (p:classes) RETURN p") - property_list = [] - flag = 0 - for objects in reply: - for obj in objects: - if flag == 0: - string = obj.decode('utf-8') - map_string = map(str.strip, string.split(',')) - property_list = list(map_string) - check = property_list.pop() - property_list.append(check.replace("\x00", "")) - print(property_list) - flag += 1 - break - if ("p.id" in property_list and - "p.operations" in property_list and - "p.properties" in property_list and - "p.type" in property_list): - return True - else: - return False - - -class TestRedisStructure(unittest.TestCase): - test_redis = Tests() - - @classmethod - def setUpClass(cls): - cls.test_database=redis.StrictRedis(host='localhost', port=6379, db=5) - cls.test_database.set("foo","bar") - cls.test_database.set("hydra","redis") - print("setUpClass db=5 keys:",cls.test_database.keys()) - - def test_entrypoint(self): - self.assertTrue(self.test_redis.entry_point()) - - def test_collectionEndpoints(self): - self.assertTrue(self.test_redis.collection_endpoints()) - - def test_classEndpoints(self): - self.assertTrue(self.test_redis.class_endpoints()) - - @classmethod - def tearDownClass(cls): - cls.test_database.flushdb() - print("tearDownClass db=5 keys:",cls.test_database.get("foo")) - -if __name__ == '__main__': - unittest.main()