From 2b71638ec407d64f2aa06292dfaa610a9fd86fe1 Mon Sep 17 00:00:00 2001 From: sandeepsajan0 Date: Sat, 16 Jun 2018 15:55:15 +0530 Subject: [PATCH 01/15] add lazyload graph --- hydra-redis/lazyload_graph.py | 135 ++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 hydra-redis/lazyload_graph.py diff --git a/hydra-redis/lazyload_graph.py b/hydra-redis/lazyload_graph.py new file mode 100644 index 0000000..60a068c --- /dev/null +++ b/hydra-redis/lazyload_graph.py @@ -0,0 +1,135 @@ +import redis +from redisgraph import Graph, Node, Edge +import urllib.request +import json +from graphviz import Digraph +import hydrus +from hydrus.hydraspec import doc_maker + + +def final_file(url): + """ Open the given url and read and load the Json data.""" + response = urllib.request.urlopen(url) + return json.loads(response.read().decode('utf-8')) + + +def addNode(node_label, node_alias, node_properties): + """Add node to redis_graph and return the same node""" + node = Node( + label=node_label, + alias=node_alias, + properties=node_properties) + redis_graph.add_node(node) + return node + + +def addEdge(src_node, predicate, dest_node): + """ Add an edge in between src_node and dest_node""" + edge = Edge(src_node, predicate, dest_node) + redis_graph.add_edge(edge) + + +def apistructure_collection(collection_endpoints, entrypoint_node): + """ Getting the properties from APIdoc for collection endpoints""" + for endpoint in collection_endpoints: + endpoint_method = [] + node_properties = {} + for support_operation in api_doc.collections[ + endpoint][ + "collection"].supportedOperation: + endpoint_method.append(support_operation.method) + node_properties["operations"] = str(endpoint_method) + node_properties["@id"] = str(collection_endpoints[endpoint]) + node_properties["@type"] = str(endpoint) + dest_node = addNode("collection", endpoint, node_properties) + addEdge(entrypoint_node, "hasCollection", dest_node) + + +def apistructure_classes(class_endpoints, entrypoint_node): + """ Getting the properties from APIdoc for class endpoints""" + for endpoint in class_endpoints: + node_properties = {} + supported_properties_list = [] + endpoint_method = [] + node_properties["@id"] = str(class_endpoints[endpoint]) + node_properties["@type"] = str(endpoint) + for support_operation in api_doc.parsed_classes[ + endpoint][ + "class"].supportedOperation: + endpoint_method.append(support_operation.method) + node_properties["operations"] = str(endpoint_method) + for support_property in api_doc.parsed_classes[ + endpoint][ + "class"].supportedProperty: + supported_properties_list.append(support_property.title) + node_properties["properties"] = str(supported_properties_list) + dest_node = addNode("class", endpoint, node_properties) + addEdge(entrypoint_node, "hasClass", dest_node) + + +def get_apistructure(entrypoint_node): + """ It breaks the endpoint into two parts collection and classes""" + collection_endpoints = {} + class_endpoints = {} + collection = 0 + classes = 0 + print("split entrypoint into 2 types of endpoints collection and classes") + for support_property in api_doc.entrypoint.entrypoint.supportedProperty: + if isinstance( + support_property, + hydrus.hydraspec.doc_writer.EntryPointClass): + class_endpoints[support_property.name] = support_property.id_ + collection = 1 + if isinstance( + support_property, + hydrus.hydraspec.doc_writer.EntryPointCollection): + collection_endpoints[support_property.name] = support_property.id_ + classes = 1 + + print("class_endpoints", class_endpoints) + print("collection_endpoints", collection_endpoints) + if classes == 1: + apistructure_classes(class_endpoints, entrypoint_node) + + if collection == 1: + apistructure_collection( + collection_endpoints, + entrypoint_node) + + +def get_entrypoint(): + """Getting tha properties for entrypoint""" + entrypoint_properties = {} + entrypoint_properties["@id"] = str("vocab:Entrypoint") + entrypoint_properties["url"] = str( + api_doc.entrypoint.url) + str(api_doc.entrypoint.api) + entrypoint_properties["supportedOperation"] = "GET" + entrypoint_node = addNode("id", "Entrypoint", entrypoint_properties) + return get_apistructure(entrypoint_node) + + +def main(user, url): + """ Setup the Redis connection and get api_doc with the help of hydrus""" + redis_con = redis.Redis(host='localhost', port=6379, db=0) + global redis_graph + redis_graph = Graph(user, redis_con) + apidoc = final_file(url + "/vocab") + global api_doc + api_doc = doc_maker.create_doc(apidoc) + return get_entrypoint() + + +if __name__ == "__main__": + user = input("user>>") + url = input("url>>") + print("lazyloading..... of graph") + main(user, url) + redis_graph.commit() +# for node in redis_graph.nodes.values(): +# print(node) + # uncomment below lines for view the graph +# g = Digraph('redis_graph', filename='hydra_graph.gv') +# # using graphviz for visualization of graph stored in redis +# for edge in redis_graph.edges: +# g.edge(edge.src_node.alias, edge.dest_node.alias) +# g.view() From d109f5c0d681c5704de3763f6fac4010e7a001bc Mon Sep 17 00:00:00 2001 From: sandeepsajan0 Date: Mon, 18 Jun 2018 02:14:58 +0530 Subject: [PATCH 02/15] lazyload graph --- hydra-redis/classes_objects.py | 94 +++++++++++++++++------------ hydra-redis/collections_endpoint.py | 60 +++++++++++------- hydra-redis/hydra_graph.py | 5 +- 3 files changed, 96 insertions(+), 63 deletions(-) diff --git a/hydra-redis/classes_objects.py b/hydra-redis/classes_objects.py index d72340c..61437b0 100644 --- a/hydra-redis/classes_objects.py +++ b/hydra-redis/classes_objects.py @@ -6,8 +6,9 @@ class ClassEndpoints: """Contains all the classes endpoint and the objects""" - def __init__(self, redis_graph): + def __init__(self, redis_graph,class_endpoints): self.redis_graph = redis_graph + self.class_endpoints = class_endpoints def addNode(self, label1, alias1, properties1): """Add node to the redis graph""" @@ -38,7 +39,6 @@ def objects_property( objects_node, new_list, no_endpoint_property, - entrypoint_node, api_doc): """Nodes for every that property which is itself an object""" print("for the property which is an object, should be a node") @@ -66,11 +66,57 @@ def objects_property( # set edge between the object and its parent object if endpoint_prop: self.objects_property( - object_node, endpoint_prop, entrypoint_node, api_doc) + object_node, endpoint_prop, api_doc) + + def load_from_server( + base_url, + class_endpoint_id, + api_doc, + ): + """Loads data from class endpoints like its properties values""" + print("check endpoint url....loading data") + members ={} + endpoint_property= [] + new_url = base_url + \ + class_endpoint_id + # url for the classes endpoint + print(new_url) + response = urllib.request.urlopen(new_url) + new_file = json.loads(response.read().decode('utf-8')) + #retreiving data for the classes endpoint from server + for support_property in api_doc.parsed_classes[ + endpoint][ + "class"].supportedProperty: + if endpoint != support_property.title and support_property.title not in self.class_endpoints: + if support_property.title in api_doc.parsed_classes: + endpoint_property.append(support_property.title) + + if support_property.title in new_file: + if isinstance(new_file[support_property.title], str): + member[support_property.title] = str( + new_file[support_property.title].replace(" ", "")) + else: + no_endpoint_property=new_file[support_property.title] + else: + member[support_property.title] = "null" + for node in self.redis_graph.nodes.values(): + if node.alias == endpoint: + node.properties["property_value"]= str(member) + #update the properties of the node + self.redis_graph.commit() + class_object_node = node + print (class_object_node) + # set edge between the entrypoint and the class endpoint/object + if endpoint_property: + self.objects_property( + class_object_node, + endpoint_property, + no_endpoint_property, + api_doc) + def endpointclasses( self, - class_endpoints, entrypoint_node, api_doc, base_url): @@ -78,23 +124,11 @@ def endpointclasses( print("classes endpoint or accessing classes") endpoint_property_list = {} # contain all endpoints which have other endpoints as a property. - for endpoint in class_endpoints: + for endpoint in self.class_endpoints: supported_properties_list=[] node_properties = {} # node_properties is used for set the properties of node. - member = {} - # member contain the properties of class endpoint. - endpoint_property = [] - # it contains the property which is a non-endpoint object or class. property_list = [] - print("check endpoint url....loading data") - new_url = base_url + \ - class_endpoints[endpoint].replace("vocab:EntryPoint", "") - # url for the classes endpoint - print(new_url) - response = urllib.request.urlopen(new_url) - new_file = json.loads(response.read().decode('utf-8')) - # retreiving data for the classes endpoint from server node_properties["operations"] = self.get_operation( api_doc, endpoint) # store the operations for the endpoint @@ -104,36 +138,18 @@ def endpointclasses( "class"].supportedProperty: supported_properties_list.append(support_property.title) if endpoint != support_property.title: - if support_property.title in class_endpoints: + if support_property.title in self.class_endpoints: property_list.append(support_property.title) - elif support_property.title in api_doc.parsed_classes: - endpoint_property.append(support_property.title) - if support_property.title in new_file: - if isinstance(new_file[support_property.title], str): - member[support_property.title] = str( - new_file[support_property.title].replace(" ", "")) - else: - no_endpoint_property=new_file[support_property.title] - else: - member[support_property.title] = "null" endpoint_property_list[endpoint] = property_list - # member is using for storing the fetched data in node. - node_properties["@id"] = str(new_file["@id"]) - node_properties["@type"] = str(new_file["@type"]) - node_properties["property_value"] = str(member) + node_properties["@id"] = str(self.class_endpoints[endpoint]) + node_properties["@type"] = str(endpoint) node_properties["properties"]= str(supported_properties_list) class_object_node = self.addNode( "classes", str(endpoint), node_properties) + print(class_object_node) self.addEdge(entrypoint_node, "has" + endpoint, class_object_node) # set edge between the entrypoint and the class endpoint/object - if endpoint_property: - self.objects_property( - class_object_node, - endpoint_property, - no_endpoint_property, - entrypoint_node, - api_doc) # for connect the nodes to endpoint which have endpoint as a property. if endpoint_property_list: for endpoint_property in endpoint_property_list: diff --git a/hydra-redis/collections_endpoint.py b/hydra-redis/collections_endpoint.py index 00da31f..8565b73 100644 --- a/hydra-redis/collections_endpoint.py +++ b/hydra-redis/collections_endpoint.py @@ -21,7 +21,6 @@ def collectionobjects( endpoint_collection_node, endpoint_list, new_url, - entrypoint_node, api_doc, url): """Creating nodes for all objects stored in collection.""" @@ -103,11 +102,44 @@ def collectionobjects( collection_object_node, no_endpoint_list, no_endpoint_property, - entrypoint_node, api_doc) else: print("NO MEMBERS") + + def load_from_server( + self, + collection_endpoint_id, + endpoint, + api_doc, + url): + """Load data or members from collection endpoint""" + print( + "check url for endpoint", + url + + collection_endpoint_id) + new_url = url + \ + collection_endpoint_id + # url for every collection endpoint + new_file = self.fetch_data(new_url) + #retrieving the objects from the collection endpoint + for node in self.redis_graph.nodes.values(): + if node.alias == endpoint: + node.properties["members"] = str(new_file["members"]) + #update the properties of node by its members + self.redis_graph.commit() + endpoint_collection_node = node + print (endpoint_collection_node) + + self.collectionobjects( + endpoint_collection_node, + new_file["members"], + new_url, + api_doc, + url + ) + + def endpointCollection( self, collection_endpoint, @@ -116,40 +148,24 @@ def endpointCollection( url): """It makes a node for every collection endpoint.""" print("accessing every collection in entrypoint") - clas = ClassEndpoints(self.redis_graph) + clas = ClassEndpoints(self.redis_graph,self.class_endpoints) for endpoint in collection_endpoint: endpoint_method = [] node_properties = {} - print( - "check url for endpoint", - url + - collection_endpoint[endpoint].replace("vocab:EntryPoint", "")) - new_url = url + \ - collection_endpoint[endpoint].replace("vocab:EntryPoint", "") - # url for every collection endpoint for support_operation in api_doc.collections[ endpoint][ "collection"].supportedOperation: endpoint_method.append(support_operation.method) node_properties["operations"] = str(endpoint_method) # all the operations for the collection endpoint is stored in - print("supportedOperations",node_properties["operations"]) +# print("supportedOperations",node_properties["operations"]) node_properties["@id"] = str(collection_endpoint[endpoint]) - new_file = self.fetch_data(new_url) - # retrieving the objects from the collection endpoint - node_properties["members"] = str(new_file["members"]) + node_properties["@type"] = str(endpoint) endpoint_collection_node = clas.addNode( "collection", endpoint, node_properties) + print (endpoint_collection_node) clas.addEdge( entrypoint_node, "has_collection", endpoint_collection_node) # set an edge between the entrypoint and collection endpoint - self.collectionobjects( - endpoint_collection_node, - new_file["members"], - new_url, - entrypoint_node, - api_doc, - url - ) diff --git a/hydra-redis/hydra_graph.py b/hydra-redis/hydra_graph.py index f60f435..62f395f 100644 --- a/hydra-redis/hydra_graph.py +++ b/hydra-redis/hydra_graph.py @@ -37,8 +37,8 @@ def get_apistructure(entrypoint_node, api_doc): print("class_endpoints", class_endpoints) print("collection_endpoints", collection_endpoints) if classes == 1: - clas = ClassEndpoints(redis_graph) - clas.endpointclasses(class_endpoints, entrypoint_node, api_doc, url) + clas = ClassEndpoints(redis_graph,class_endpoints) + clas.endpointclasses(entrypoint_node, api_doc, url) if collection == 1: coll = CollectionEndpoints(redis_graph, class_endpoints) @@ -70,6 +70,7 @@ def get_endpoints(api_doc): redis_graph = Graph("apidoc", redis_con) url = "http://35.224.198.158:8080/api" # you also can use https://storage.googleapis.com/api2/api as url + print("loading... of graph") apidoc = final_file(url + "/vocab") api_doc = doc_maker.create_doc(apidoc) get_endpoints(api_doc) From 950b24b09e64f9c2410b2a96f160f336cd0ab04e Mon Sep 17 00:00:00 2001 From: sandeepsajan0 Date: Mon, 18 Jun 2018 02:26:13 +0530 Subject: [PATCH 03/15] update lazyload graph --- hydra-redis/lazyload_graph.py | 135 ---------------------------------- 1 file changed, 135 deletions(-) delete mode 100644 hydra-redis/lazyload_graph.py diff --git a/hydra-redis/lazyload_graph.py b/hydra-redis/lazyload_graph.py deleted file mode 100644 index 60a068c..0000000 --- a/hydra-redis/lazyload_graph.py +++ /dev/null @@ -1,135 +0,0 @@ -import redis -from redisgraph import Graph, Node, Edge -import urllib.request -import json -from graphviz import Digraph -import hydrus -from hydrus.hydraspec import doc_maker - - -def final_file(url): - """ Open the given url and read and load the Json data.""" - response = urllib.request.urlopen(url) - return json.loads(response.read().decode('utf-8')) - - -def addNode(node_label, node_alias, node_properties): - """Add node to redis_graph and return the same node""" - node = Node( - label=node_label, - alias=node_alias, - properties=node_properties) - redis_graph.add_node(node) - return node - - -def addEdge(src_node, predicate, dest_node): - """ Add an edge in between src_node and dest_node""" - edge = Edge(src_node, predicate, dest_node) - redis_graph.add_edge(edge) - - -def apistructure_collection(collection_endpoints, entrypoint_node): - """ Getting the properties from APIdoc for collection endpoints""" - for endpoint in collection_endpoints: - endpoint_method = [] - node_properties = {} - for support_operation in api_doc.collections[ - endpoint][ - "collection"].supportedOperation: - endpoint_method.append(support_operation.method) - node_properties["operations"] = str(endpoint_method) - node_properties["@id"] = str(collection_endpoints[endpoint]) - node_properties["@type"] = str(endpoint) - dest_node = addNode("collection", endpoint, node_properties) - addEdge(entrypoint_node, "hasCollection", dest_node) - - -def apistructure_classes(class_endpoints, entrypoint_node): - """ Getting the properties from APIdoc for class endpoints""" - for endpoint in class_endpoints: - node_properties = {} - supported_properties_list = [] - endpoint_method = [] - node_properties["@id"] = str(class_endpoints[endpoint]) - node_properties["@type"] = str(endpoint) - for support_operation in api_doc.parsed_classes[ - endpoint][ - "class"].supportedOperation: - endpoint_method.append(support_operation.method) - node_properties["operations"] = str(endpoint_method) - for support_property in api_doc.parsed_classes[ - endpoint][ - "class"].supportedProperty: - supported_properties_list.append(support_property.title) - node_properties["properties"] = str(supported_properties_list) - dest_node = addNode("class", endpoint, node_properties) - addEdge(entrypoint_node, "hasClass", dest_node) - - -def get_apistructure(entrypoint_node): - """ It breaks the endpoint into two parts collection and classes""" - collection_endpoints = {} - class_endpoints = {} - collection = 0 - classes = 0 - print("split entrypoint into 2 types of endpoints collection and classes") - for support_property in api_doc.entrypoint.entrypoint.supportedProperty: - if isinstance( - support_property, - hydrus.hydraspec.doc_writer.EntryPointClass): - class_endpoints[support_property.name] = support_property.id_ - collection = 1 - if isinstance( - support_property, - hydrus.hydraspec.doc_writer.EntryPointCollection): - collection_endpoints[support_property.name] = support_property.id_ - classes = 1 - - print("class_endpoints", class_endpoints) - print("collection_endpoints", collection_endpoints) - if classes == 1: - apistructure_classes(class_endpoints, entrypoint_node) - - if collection == 1: - apistructure_collection( - collection_endpoints, - entrypoint_node) - - -def get_entrypoint(): - """Getting tha properties for entrypoint""" - entrypoint_properties = {} - entrypoint_properties["@id"] = str("vocab:Entrypoint") - entrypoint_properties["url"] = str( - api_doc.entrypoint.url) + str(api_doc.entrypoint.api) - entrypoint_properties["supportedOperation"] = "GET" - entrypoint_node = addNode("id", "Entrypoint", entrypoint_properties) - return get_apistructure(entrypoint_node) - - -def main(user, url): - """ Setup the Redis connection and get api_doc with the help of hydrus""" - redis_con = redis.Redis(host='localhost', port=6379, db=0) - global redis_graph - redis_graph = Graph(user, redis_con) - apidoc = final_file(url + "/vocab") - global api_doc - api_doc = doc_maker.create_doc(apidoc) - return get_entrypoint() - - -if __name__ == "__main__": - user = input("user>>") - url = input("url>>") - print("lazyloading..... of graph") - main(user, url) - redis_graph.commit() -# for node in redis_graph.nodes.values(): -# print(node) - # uncomment below lines for view the graph -# g = Digraph('redis_graph', filename='hydra_graph.gv') -# # using graphviz for visualization of graph stored in redis -# for edge in redis_graph.edges: -# g.edge(edge.src_node.alias, edge.dest_node.alias) -# g.view() From 646ae29d4326ef00a6ae67a1989c11f6294b3dd0 Mon Sep 17 00:00:00 2001 From: sandeepsajan0 Date: Mon, 18 Jun 2018 02:39:05 +0530 Subject: [PATCH 04/15] follow pep8 style --- hydra-redis/classes_objects.py | 39 +++++++++++++++-------------- hydra-redis/collections_endpoint.py | 32 +++++++++++------------ hydra-redis/hydra_graph.py | 2 +- 3 files changed, 37 insertions(+), 36 deletions(-) diff --git a/hydra-redis/classes_objects.py b/hydra-redis/classes_objects.py index 61437b0..3ced952 100644 --- a/hydra-redis/classes_objects.py +++ b/hydra-redis/classes_objects.py @@ -6,7 +6,7 @@ class ClassEndpoints: """Contains all the classes endpoint and the objects""" - def __init__(self, redis_graph,class_endpoints): + def __init__(self, redis_graph, class_endpoints): self.redis_graph = redis_graph self.class_endpoints = class_endpoints @@ -69,25 +69,27 @@ def objects_property( object_node, endpoint_prop, api_doc) def load_from_server( - base_url, + self, class_endpoint_id, + endpoint, api_doc, - ): + base_url): """Loads data from class endpoints like its properties values""" print("check endpoint url....loading data") - members ={} - endpoint_property= [] + member = {} + endpoint_property = [] new_url = base_url + \ class_endpoint_id # url for the classes endpoint print(new_url) response = urllib.request.urlopen(new_url) new_file = json.loads(response.read().decode('utf-8')) - #retreiving data for the classes endpoint from server + # retreiving data for the classes endpoint from server for support_property in api_doc.parsed_classes[ - endpoint][ - "class"].supportedProperty: - if endpoint != support_property.title and support_property.title not in self.class_endpoints: + endpoint][ + "class"].supportedProperty: + if (endpoint != support_property.title and + support_property.title not in self.class_endpoints): if support_property.title in api_doc.parsed_classes: endpoint_property.append(support_property.title) @@ -96,16 +98,16 @@ def load_from_server( member[support_property.title] = str( new_file[support_property.title].replace(" ", "")) else: - no_endpoint_property=new_file[support_property.title] + no_endpoint_property = new_file[support_property.title] else: member[support_property.title] = "null" for node in self.redis_graph.nodes.values(): if node.alias == endpoint: - node.properties["property_value"]= str(member) - #update the properties of the node + node.properties["property_value"] = str(member) + # update the properties of the node self.redis_graph.commit() class_object_node = node - print (class_object_node) + print(class_object_node) # set edge between the entrypoint and the class endpoint/object if endpoint_property: self.objects_property( @@ -114,7 +116,6 @@ def load_from_server( no_endpoint_property, api_doc) - def endpointclasses( self, entrypoint_node, @@ -125,7 +126,7 @@ def endpointclasses( endpoint_property_list = {} # contain all endpoints which have other endpoints as a property. for endpoint in self.class_endpoints: - supported_properties_list=[] + supported_properties_list = [] node_properties = {} # node_properties is used for set the properties of node. property_list = [] @@ -134,8 +135,8 @@ def endpointclasses( # store the operations for the endpoint for support_property in api_doc.parsed_classes[ - endpoint][ - "class"].supportedProperty: + endpoint][ + "class"].supportedProperty: supported_properties_list.append(support_property.title) if endpoint != support_property.title: if support_property.title in self.class_endpoints: @@ -144,7 +145,7 @@ def endpointclasses( endpoint_property_list[endpoint] = property_list node_properties["@id"] = str(self.class_endpoints[endpoint]) node_properties["@type"] = str(endpoint) - node_properties["properties"]= str(supported_properties_list) + node_properties["properties"] = str(supported_properties_list) class_object_node = self.addNode( "classes", str(endpoint), node_properties) print(class_object_node) @@ -156,7 +157,7 @@ def endpointclasses( for src_node in self.redis_graph.nodes.values(): if str(endpoint_property) == src_node.alias: for endpoints in endpoint_property_list[ - endpoint_property]: + endpoint_property]: for nodes in self.redis_graph.nodes.values(): if endpoints == nodes.alias: self.addEdge( diff --git a/hydra-redis/collections_endpoint.py b/hydra-redis/collections_endpoint.py index 8565b73..dac14aa 100644 --- a/hydra-redis/collections_endpoint.py +++ b/hydra-redis/collections_endpoint.py @@ -49,16 +49,16 @@ def collectionobjects( new_file1 = self.fetch_data(new_url1) # object data retrieving from the server for support_operation in api_doc.parsed_classes[ - endpoint["@type"] - ]["class" - ].supportedOperation: + endpoint["@type"] + ]["class" + ].supportedOperation: endpoint_method.append(support_operation.method) node_properties["operations"] = str(endpoint_method) # all the operations for the object is stored in method for support_property in api_doc.parsed_classes[ - endpoint["@type"] - ]["class" - ].supportedProperty: + endpoint["@type"] + ]["class" + ].supportedProperty: supported_property_list.append(support_property.title) if support_property.title in self.class_endpoints: endpoint_property_list.append( @@ -69,9 +69,11 @@ def collectionobjects( if support_property.title in new_file1: if isinstance(new_file1[support_property.title], str): member[support_property.title] = str( - new_file1[support_property.title].replace(" ", "")) + new_file1[ + support_property.title].replace(" ", "")) else: - no_endpoint_property = new_file1[support_property.title] + no_endpoint_property = new_file1[ + support_property.title] else: member[support_property.title] = "null" @@ -106,7 +108,6 @@ def collectionobjects( else: print("NO MEMBERS") - def load_from_server( self, collection_endpoint_id, @@ -122,15 +123,15 @@ def load_from_server( collection_endpoint_id # url for every collection endpoint new_file = self.fetch_data(new_url) - #retrieving the objects from the collection endpoint + # retrieving the objects from the collection endpoint for node in self.redis_graph.nodes.values(): if node.alias == endpoint: node.properties["members"] = str(new_file["members"]) - #update the properties of node by its members + # update the properties of node by its members self.redis_graph.commit() endpoint_collection_node = node - print (endpoint_collection_node) - + print(endpoint_collection_node) + self.collectionobjects( endpoint_collection_node, new_file["members"], @@ -139,7 +140,6 @@ def load_from_server( url ) - def endpointCollection( self, collection_endpoint, @@ -148,7 +148,7 @@ def endpointCollection( url): """It makes a node for every collection endpoint.""" print("accessing every collection in entrypoint") - clas = ClassEndpoints(self.redis_graph,self.class_endpoints) + clas = ClassEndpoints(self.redis_graph, self.class_endpoints) for endpoint in collection_endpoint: endpoint_method = [] node_properties = {} @@ -163,7 +163,7 @@ def endpointCollection( node_properties["@type"] = str(endpoint) endpoint_collection_node = clas.addNode( "collection", endpoint, node_properties) - print (endpoint_collection_node) + print(endpoint_collection_node) clas.addEdge( entrypoint_node, "has_collection", diff --git a/hydra-redis/hydra_graph.py b/hydra-redis/hydra_graph.py index 62f395f..096c754 100644 --- a/hydra-redis/hydra_graph.py +++ b/hydra-redis/hydra_graph.py @@ -37,7 +37,7 @@ def get_apistructure(entrypoint_node, api_doc): print("class_endpoints", class_endpoints) print("collection_endpoints", collection_endpoints) if classes == 1: - clas = ClassEndpoints(redis_graph,class_endpoints) + clas = ClassEndpoints(redis_graph, class_endpoints) clas.endpointclasses(entrypoint_node, api_doc, url) if collection == 1: From f71342997eca61903f39d0881cbd612700ba7c6c Mon Sep 17 00:00:00 2001 From: sandeepsajan0 Date: Fri, 22 Jun 2018 22:27:29 +0530 Subject: [PATCH 05/15] update of querying mechanism --- hydra-redis/classes_objects.py | 11 +- hydra-redis/collections_endpoint.py | 26 ++-- hydra-redis/hydra_graph.py | 25 ++-- hydra-redis/querying_mechanism.py | 184 ++++++++++++++++++++++++++++ 4 files changed, 220 insertions(+), 26 deletions(-) create mode 100644 hydra-redis/querying_mechanism.py diff --git a/hydra-redis/classes_objects.py b/hydra-redis/classes_objects.py index 3ced952..660b4f2 100644 --- a/hydra-redis/classes_objects.py +++ b/hydra-redis/classes_objects.py @@ -29,7 +29,7 @@ def get_operation(self, api_doc, endpoint): endpoint][ "class"].supportedOperation: endpoint_method.append(support_operation.method) - print("supportedOperation", endpoint_method) +# print("supportedOperation", endpoint_method) # all the operations for the object is stored in endpoint_method return str(endpoint_method) @@ -56,12 +56,12 @@ def objects_property( if support_property.title in api_doc.parsed_classes: endpoint_prop.append(support_property.title) # store all operation and property of object - node_properties["property"] = str(properties_title) + node_properties["properties"] = str(properties_title) node_properties["property_value"] = str(no_endpoint_property) node_alias = str(objects_node.alias + str(obj)).lower() # key for the node of the object node_properties["parent_id"] = str(objects_node.properties["@id"]) - object_node = self.addNode("object", node_alias, node_properties) + object_node = self.addNode(str("object"+str(objects_node.properties["@type"])), node_alias, node_properties) self.addEdge(objects_node, "has" + str(obj), object_node) # set edge between the object and its parent object if endpoint_prop: @@ -70,7 +70,6 @@ def objects_property( def load_from_server( self, - class_endpoint_id, endpoint, api_doc, base_url): @@ -79,7 +78,7 @@ def load_from_server( member = {} endpoint_property = [] new_url = base_url + \ - class_endpoint_id + endpoint # url for the classes endpoint print(new_url) response = urllib.request.urlopen(new_url) @@ -148,7 +147,7 @@ def endpointclasses( node_properties["properties"] = str(supported_properties_list) class_object_node = self.addNode( "classes", str(endpoint), node_properties) - print(class_object_node) +# print(class_object_node) self.addEdge(entrypoint_node, "has" + endpoint, class_object_node) # set edge between the entrypoint and the class endpoint/object # for connect the nodes to endpoint which have endpoint as a property. diff --git a/hydra-redis/collections_endpoint.py b/hydra-redis/collections_endpoint.py index dac14aa..7a711bf 100644 --- a/hydra-redis/collections_endpoint.py +++ b/hydra-redis/collections_endpoint.py @@ -26,7 +26,7 @@ def collectionobjects( """Creating nodes for all objects stored in collection.""" print("accesing the collection object like events or drones") if endpoint_list: - clas = ClassEndpoints(self.redis_graph) + clas = ClassEndpoints(self.redis_graph,self.class_endpoints) for endpoint in endpoint_list: node_properties = {} no_endpoint_list = [] @@ -40,11 +40,11 @@ def collectionobjects( match_obj.group(2)) entrypoint_member = endpoint["@type"].lower( ) + match_obj.group(3) - print(base_url, entrypoint_member) +# print(base_url, entrypoint_member,endpoint["@type"]) member_alias = entrypoint_member # key for the object node is memeber_alias member_id = match_obj.group(3) - print("member alias and id", member_alias, member_id) +# print("member alias and id", member_alias, member_id) new_url1 = new_url + "/" + member_id new_file1 = self.fetch_data(new_url1) # object data retrieving from the server @@ -79,13 +79,15 @@ def collectionobjects( node_properties["@id"] = str(endpoint["@id"]) node_properties["@type"] = str(endpoint["@type"]) + member[endpoint["@type"]]= str(endpoint["@id"]) node_properties["property_value"] = str(member) node_properties["properties"] = str(supported_property_list) collection_object_node = clas.addNode( - "objects", str(member_alias), node_properties) + str("objects"+str(endpoint["@type"])), str(member_alias), node_properties) # add object as a node in redis clas.addEdge(endpoint_collection_node, "has_" + str(endpoint["@type"]), collection_object_node) + # set an edge between the collection and its object print( "property of endpoint which can be class but not endpoint", @@ -105,22 +107,22 @@ def collectionobjects( no_endpoint_list, no_endpoint_property, api_doc) + self.redis_graph.commit() else: print("NO MEMBERS") def load_from_server( self, - collection_endpoint_id, endpoint, api_doc, url): """Load data or members from collection endpoint""" print( "check url for endpoint", - url + - collection_endpoint_id) - new_url = url + \ - collection_endpoint_id + url + "/" + + endpoint) + new_url = url + "/"+\ + endpoint # url for every collection endpoint new_file = self.fetch_data(new_url) # retrieving the objects from the collection endpoint @@ -130,7 +132,7 @@ def load_from_server( # update the properties of node by its members self.redis_graph.commit() endpoint_collection_node = node - print(endpoint_collection_node) +# print(endpoint_collection_node) self.collectionobjects( endpoint_collection_node, @@ -139,6 +141,8 @@ def load_from_server( api_doc, url ) +# for node in self.redis_graph.nodes.values(): +# print("\n",node.alias) def endpointCollection( self, @@ -163,7 +167,7 @@ def endpointCollection( node_properties["@type"] = str(endpoint) endpoint_collection_node = clas.addNode( "collection", endpoint, node_properties) - print(endpoint_collection_node) +# print(endpoint_collection_node) clas.addEdge( entrypoint_node, "has_collection", diff --git a/hydra-redis/hydra_graph.py b/hydra-redis/hydra_graph.py index 096c754..9430afc 100644 --- a/hydra-redis/hydra_graph.py +++ b/hydra-redis/hydra_graph.py @@ -17,7 +17,9 @@ def final_file(url): def get_apistructure(entrypoint_node, api_doc): """ It breaks the endpoint into two parts collection and classes""" + global collection_endpoints collection_endpoints = {} + global class_endpoints class_endpoints = {} collection = 0 classes = 0 @@ -34,8 +36,8 @@ def get_apistructure(entrypoint_node, api_doc): collection_endpoints[support_property.name] = support_property.id_ classes = 1 - print("class_endpoints", class_endpoints) - print("collection_endpoints", collection_endpoints) +# print("class_endpoints", class_endpoints) +# print("collection_endpoints", collection_endpoints) if classes == 1: clas = ClassEndpoints(redis_graph, class_endpoints) clas.endpointclasses(entrypoint_node, api_doc, url) @@ -64,23 +66,22 @@ def get_endpoints(api_doc): redis_graph.add_node(entrypoint_node) return get_apistructure(entrypoint_node, api_doc) - -if __name__ == "__main__": +def main(new_url,api_doc): redis_con = redis.Redis(host='localhost', port=6379) + global redis_graph + global url + url=new_url redis_graph = Graph("apidoc", redis_con) - url = "http://35.224.198.158:8080/api" # you also can use https://storage.googleapis.com/api2/api as url print("loading... of graph") - apidoc = final_file(url + "/vocab") - api_doc = doc_maker.create_doc(apidoc) get_endpoints(api_doc) print("commiting") redis_graph.commit() # creating whole the graph in redis print("done!!!!") # uncomment below 2 lines for getting nodes for whole graph -# for node in redis_graph.nodes: -# print("\n",node) +# for node in redis_graph.nodes.values(): +# print("\n",node.alias) # uncomment the below lines for show the graph stored in redis # g = Digraph('redis_graph', filename='hydra_graph.gv') # # using graphviz for visualization of graph stored in redis @@ -88,3 +89,9 @@ def get_endpoints(api_doc): # g.edge(edge.src_node.alias, edge.dest_node.alias) # g.view() # #see the graph generated by graphviz + +if __name__ == "__main__": + url = "http://35.224.198.158:8080/api" + apidoc = final_file(url + "/vocab") + api_doc = doc_maker.create_doc(apidoc) + main(url,api_doc) diff --git a/hydra-redis/querying_mechanism.py b/hydra-redis/querying_mechanism.py new file mode 100644 index 0000000..1fff55d --- /dev/null +++ b/hydra-redis/querying_mechanism.py @@ -0,0 +1,184 @@ +import redis +import hydra_graph as graph +from hydrus.hydraspec import doc_maker +import hydrus +from collections_endpoint import CollectionEndpoints +import urllib.request +import json + +def final_file(url): + """ Open the given url and read and load the Json data.""" + response = urllib.request.urlopen(url) + return json.loads(response.read().decode('utf-8')) + +def split_data(count,objects): + for obj in objects: + string = obj.decode('utf-8') + map_string = map(str.strip,string.split(',')) + property_list = list(map_string) + check = property_list.pop() + property_list.append(check.replace("\x00","")) + print (property_list) + +def endpoints_query(query): + r = redis.StrictRedis(host='localhost', port=6379, db=0) + if query == "endpoint": + reply = r.execute_command('GRAPH.QUERY', + 'apidoc', "MATCH (p:classes) RETURN p") + \ + r.execute_command('GRAPH.QUERY', + 'apidoc', "MATCH (p:collection) RETURN p") + count = 0 + + if query == "classEndpoint": + reply = r.execute_command('GRAPH.QUERY', + 'apidoc','MATCH (p:classes)' 'RETURN p') + count = 0 + + if query == "collectionEndpoint": +# reply = r.execute_command('GRAPH.QUERY','apidoc','RETURN ',query,'') + reply = r.execute_command('GRAPH.QUERY', + 'apidoc','MATCH (p:collection) RETURN p') + count = 2 + + for objects in reply: + count+=1 + if count==1: + print("\nclasses endpoint\n") + if count==3: + print("\ncollection endpoint\n") + if count%2!=0: + split_data(count,objects) +# print (reply) + +def operation_and_property_query(query): + r = redis.StrictRedis(host='localhost', port=6379, db=0) + endpoint, query = query.split(" ") +# print(endpoint) + if "objects" in endpoint: + endpoint = endpoint.replace("Collection","") + id_ = endpoint + print(id_,endpoint) + endpoint = endpoint.replace("objects","") + reply = r.execute_command('GRAPH.QUERY', + 'apidoc','MATCH ( p:{} ) WHERE (p.type = "{}") RETURN p.{}'.format(id_,endpoint,query)) + print("objects") + count = 0 + elif "object" in endpoint: + index = endpoint.find("Collection") + id_ = endpoint[5:index] +# print (id_) + endpoint = endpoint.replace("object","") +# endpoint = "/api/" + endpoint + print("object") + reply = r.execute_command('GRAPH.QUERY', + 'apidoc','MATCH ( p:{}) WHERE (p.parent_id = "{}") RETURN p.{}'.format(id_,endpoint,query)) + count = 0 + elif "Collection" in endpoint: + reply = r.execute_command('GRAPH.QUERY', + 'apidoc','MATCH ( p:collection ) WHERE (p.type="{}") RETURN p.{}'.format(endpoint,query)) + count = 2 + print("collection_endpoint") + else: + reply = r.execute_command('GRAPH.QUERY', + 'apidoc','MATCH ( p:classes ) WHERE (p.type="{}") RETURN p.{}'.format(endpoint,query)) + count = 0 + print("class_endpoint") + for objects in reply: + count+=1 + if count%2!=0: + split_data(count,objects) + + +def collection_endpoint_members(endpoint): + collect = CollectionEndpoints(graph.redis_graph,graph.class_endpoints) + r = redis.StrictRedis(host='localhost', port=6379, db=0) + endpoint = endpoint.replace(" members","") +# collect.load_from_server(endpoint,api_doc,url) + query = "objects"+endpoint[:-10] +# print(query) + reply = r.execute_command('GRAPH.QUERY', + 'apidoc','MATCH (p:{}) WHERE (p.type = "{}") RETURN p.id'.format(query,endpoint[:-10])) +# print(reply) + if len(reply[0])==1: + collect.load_from_server(endpoint,api_doc,url) + reply = r.execute_command('GRAPH.QUERY', + 'apidoc','MATCH (p:{}) WHERE (p.type = "{}") RETURN p.id'.format(query,endpoint[:-10])) + count=2 + print("members") + for objects in reply: + count+=1 + if count%2!=0: + split_data(count,objects) + + +#def compare_property(query): +# endpoint_type, query1 = query.split(" ",1) +## query = list(map(str.strip, query.split(' '))) +# r = redis.StrictRedis(host='localhost', port=6379, db=0) +# query = endpoint_type +# endpoint_type = "objects" + endpoint_type +# print(endpoint_type,query1) +# reply = r.execute_command('GRAPH.QUERY', +# 'apidoc','MATCH ( p:{} ) WHERE (p.type= "{}") RETURN p.property_value'.format(endpoint_type,query)) +# count = 0 +# for objects in reply: +# count+=1 +# if count%2!=0: +# for obj in objects: +# string = obj.decode('utf-8') +# print (string) +# map_string = map(str.strip,string.split(',')) +# property_list = list(map_string) +# check = property_list.pop() +# property_list.append(check.replace("\x00","")) +# for query1 in dict(property_list[0]): +# print(query1) +# print (property_list[0]) +# print(reply) + + +def main(): + initialize= 0 + while(1): + if initialize ==0: + print("just initialize") + global url + url =input("url>>>") + apidoc = final_file(url + "/vocab") + global api_doc + api_doc = doc_maker.create_doc(apidoc) + graph.main(url,api_doc) + initialize+=1 + elif initialize >0: +# print("for query write your query in queryformat") + print("for exit enter EXIT or write query in queryformat") + query = input("query>>>") + if query == "EXIT": + break + else: + query = query.replace("show ","") + if query=="endpoint" or query == "collectionEndpoint" or query == "classEndpoint": + endpoints_query(query) + elif "members" in query: + collection_endpoint_members(query) + elif "properties" in query or "operations" in query or "property_value" in query or "type" in query or "id" in query: + operation_and_property_query(query) +# else: +# compare_property(query) + + +if __name__ =="__main__": + print ("querying format") + print ("for endpoint:- show endpoint") + print ("for class_endpoint:- show classEndpoint") + print ("for collection_endpoint:- show collectionEndpoint") + print ("for members of collection_endpoint:-show members") + print ("for properties of any member:- show object properties ") + print ("for properties of objects:- show objects properties") + print ("for collection or class properties:- show properties") + main() +# query = input() +# query = query.replace("show ","") +# operation_and_property_query(query) +## endpoints_query(query) +# class_endpoint_query("http://35.224.198.158:8080/api","classEndpoint") From fd2e1f48dbbc66fc6f4cd3874b8b3222932e7b3a Mon Sep 17 00:00:00 2001 From: sandeepsajan0 Date: Wed, 27 Jun 2018 00:57:49 +0530 Subject: [PATCH 06/15] update coding style and use try-except --- hydra-redis/classes_objects.py | 10 + hydra-redis/collections_endpoint.py | 10 +- hydra-redis/hydra_graph.py | 1 - hydra-redis/querying_mechanism.py | 427 ++++++++++++++++++---------- 4 files changed, 291 insertions(+), 157 deletions(-) diff --git a/hydra-redis/classes_objects.py b/hydra-redis/classes_objects.py index 660b4f2..eeed6c5 100644 --- a/hydra-redis/classes_objects.py +++ b/hydra-redis/classes_objects.py @@ -34,6 +34,16 @@ def get_operation(self, api_doc, endpoint): return str(endpoint_method) + def property_value(self,src_node,member,id_): + id_=str("objects"+id_) + node_properties = {} + for members in member: + node_properties[members] = str(member[members]) + node = Node(label = "properties",alias = str(id_), properties = node_properties) + self.redis_graph.add_node(node) + edge = Edge(src_node,"hasproperty",node) + self.redis_graph.add_edge(edge) + def objects_property( self, objects_node, diff --git a/hydra-redis/collections_endpoint.py b/hydra-redis/collections_endpoint.py index 7a711bf..01bab05 100644 --- a/hydra-redis/collections_endpoint.py +++ b/hydra-redis/collections_endpoint.py @@ -44,7 +44,7 @@ def collectionobjects( member_alias = entrypoint_member # key for the object node is memeber_alias member_id = match_obj.group(3) -# print("member alias and id", member_alias, member_id) + print("member alias and id", member_alias.capitalize(), member_id) new_url1 = new_url + "/" + member_id new_file1 = self.fetch_data(new_url1) # object data retrieving from the server @@ -83,7 +83,7 @@ def collectionobjects( node_properties["property_value"] = str(member) node_properties["properties"] = str(supported_property_list) collection_object_node = clas.addNode( - str("objects"+str(endpoint["@type"])), str(member_alias), node_properties) + str("objects"+str(endpoint["@type"])), str(member_alias.capitalize()), node_properties) # add object as a node in redis clas.addEdge(endpoint_collection_node, "has_" + str(endpoint["@type"]), collection_object_node) @@ -93,6 +93,7 @@ def collectionobjects( "property of endpoint which can be class but not endpoint", no_endpoint_list ) + clas.property_value(collection_object_node,member,member_alias.capitalize()) if endpoint_property_list: for endpoint_property in endpoint_property_list: for nodes in self.redis_graph.nodes.values(): @@ -107,7 +108,7 @@ def collectionobjects( no_endpoint_list, no_endpoint_property, api_doc) - self.redis_graph.commit() +# self.redis_graph.commit() else: print("NO MEMBERS") @@ -130,7 +131,7 @@ def load_from_server( if node.alias == endpoint: node.properties["members"] = str(new_file["members"]) # update the properties of node by its members - self.redis_graph.commit() +# self.redis_graph.commit() endpoint_collection_node = node # print(endpoint_collection_node) @@ -141,6 +142,7 @@ def load_from_server( api_doc, url ) + self.redis_graph.commit() # for node in self.redis_graph.nodes.values(): # print("\n",node.alias) diff --git a/hydra-redis/hydra_graph.py b/hydra-redis/hydra_graph.py index 9430afc..cc9a332 100644 --- a/hydra-redis/hydra_graph.py +++ b/hydra-redis/hydra_graph.py @@ -72,7 +72,6 @@ def main(new_url,api_doc): global url url=new_url redis_graph = Graph("apidoc", redis_con) - # you also can use https://storage.googleapis.com/api2/api as url print("loading... of graph") get_endpoints(api_doc) print("commiting") diff --git a/hydra-redis/querying_mechanism.py b/hydra-redis/querying_mechanism.py index 1fff55d..686b0a0 100644 --- a/hydra-redis/querying_mechanism.py +++ b/hydra-redis/querying_mechanism.py @@ -1,170 +1,293 @@ import redis import hydra_graph as graph -from hydrus.hydraspec import doc_maker import hydrus -from collections_endpoint import CollectionEndpoints import urllib.request import json +from hydrus.hydraspec import doc_maker +from urllib.error import URLError, HTTPError +from collections_endpoint import CollectionEndpoints -def final_file(url): - """ Open the given url and read and load the Json data.""" - response = urllib.request.urlopen(url) - return json.loads(response.read().decode('utf-8')) - -def split_data(count,objects): - for obj in objects: - string = obj.decode('utf-8') - map_string = map(str.strip,string.split(',')) - property_list = list(map_string) - check = property_list.pop() - property_list.append(check.replace("\x00","")) - print (property_list) - -def endpoints_query(query): - r = redis.StrictRedis(host='localhost', port=6379, db=0) - if query == "endpoint": - reply = r.execute_command('GRAPH.QUERY', - 'apidoc', "MATCH (p:classes) RETURN p") + \ - r.execute_command('GRAPH.QUERY', - 'apidoc', "MATCH (p:collection) RETURN p") - count = 0 - if query == "classEndpoint": - reply = r.execute_command('GRAPH.QUERY', - 'apidoc','MATCH (p:classes)' 'RETURN p') - count = 0 +class HandleData: + """ + Handle data is used to play with data. + It have two functions load_data and show_data. + Work of load_data is to fetch the data from server and + Work of show_data is to show the data of Redis in readable format. + """ + def load_data(self,url): + """ + Load the data for the given url and return it. + Also handle with the HTTPError, it prints the error + """ + try: + response = urllib.request.urlopen(url) + except HTTPError as e: + print('Error code: ', e.code) + return ("error") + except URLError as e: + print('Reason: ', e.reason) + return ("error") + else: + return json.loads(response.read().decode('utf-8')) - if query == "collectionEndpoint": -# reply = r.execute_command('GRAPH.QUERY','apidoc','RETURN ',query,'') - reply = r.execute_command('GRAPH.QUERY', - 'apidoc','MATCH (p:collection) RETURN p') - count = 2 - - for objects in reply: - count+=1 - if count==1: - print("\nclasses endpoint\n") - if count==3: - print("\ncollection endpoint\n") - if count%2!=0: - split_data(count,objects) -# print (reply) - -def operation_and_property_query(query): - r = redis.StrictRedis(host='localhost', port=6379, db=0) - endpoint, query = query.split(" ") -# print(endpoint) - if "objects" in endpoint: - endpoint = endpoint.replace("Collection","") - id_ = endpoint - print(id_,endpoint) - endpoint = endpoint.replace("objects","") - reply = r.execute_command('GRAPH.QUERY', - 'apidoc','MATCH ( p:{} ) WHERE (p.type = "{}") RETURN p.{}'.format(id_,endpoint,query)) - print("objects") + def show_data(self,get_data): + """ + Make the given data readable, because now it is in binary string form. + Count is using for avoid stuffs like query internal execution time. + """ count = 0 - elif "object" in endpoint: - index = endpoint.find("Collection") - id_ = endpoint[5:index] -# print (id_) + for objects in get_data: + count+=1 + if count%2!=0: + for obj in objects: + string = obj.decode('utf-8') + map_string = map(str.strip,string.split(',')) + property_list = list(map_string) + check = property_list.pop() + property_list.append(check.replace("\x00","")) + print (property_list) + + +class RedisProxy: + """ + RedisProxy is used for make a connection to the Redis. + """ + def __init__(self): + self.connection = redis.StrictRedis(host='localhost', port=6379, db=0) + + def get_connection(self): + return self.connection + + +class EndpointQuery: + """ + EndpointQuery is used for get the endpoints from the Redis. + """ + def __init__(self): + self.redis_connection = RedisProxy() + self.handle_data = HandleData() + self.connection = self.redis_connection.get_connection() + self._data = self.handle_data + + def get_allEndpoints(self,query): + """ + It will return both type(class and collection) of endpoints. + """ + get_data = self.connection.execute_command('GRAPH.QUERY','apidoc', "MATCH (p:classes) RETURN p") + \ + self.connection.execute_command ('GRAPH.QUERY','apidoc', "MATCH (p:collection) RETURN p") + print("classEndpoints + CollectionEndpoints") + + return self._data.show_data(get_data) + + def get_classEndpoints(self,query): + """ + It will return all class Endpoints. + """ + get_data = self.connection.execute_command('GRAPH.QUERY','apidoc', "MATCH (p:classes) RETURN p") + + print("classEndpoints") + + return self._data.show_data(get_data) + + def get_collectionEndpoints(self,query): + """ + It will returns all collection Endpoints. + """ + get_data = self.connection.execute_command ('GRAPH.QUERY','apidoc', "MATCH (p:collection) RETURN p") + + print("collectoinEndpoints") + + return self._data.show_data(get_data) + + +class CollectionmembersQuery: + """ + CollectionmembersQuery is used for get members of any collectionendpoint. + Once it get the data from the server and store it in Redis. + And after that it can query from Redis memory. + Check_list is using for track which collection endpoint data is in Redis. + """ + def __init__(self,api_doc,url): + self.redis_connection = RedisProxy() + self.handle_data = HandleData() + self.connection = self.redis_connection.get_connection() + self._data = self.handle_data + self.collect = CollectionEndpoints(graph.redis_graph, + graph.class_endpoints) + + self.api_doc = api_doc + self.url = url + + def data_from_server(self,endpoint): + """ + Load data from the server for first time. + """ + self.collect.load_from_server(endpoint,self.api_doc,self.url) + + get_data = self.connection.execute_command('GRAPH.QUERY', + 'apidoc','MATCH (p:collection) WHERE (p.type = "{}") RETURN p.members'.format(endpoint)) + return get_data + + def get_members(self,query): + """ + Gets Data from the Redis. + """ + endpoint = query.replace(" members","") + print(check_list) + if endpoint in check_list: + get_data = self.connection.execute_command('GRAPH.QUERY', + 'apidoc','MATCH (p:collection) WHERE (p.type = "{}") RETURN p.members'.format(endpoint)) + + else: + check_list.append(endpoint) + print(check_list) + get_data = self.data_from_server(endpoint) + + print(endpoint," members") + return self._data.show_data(get_data) + + + +class PropertiesQuery: + """ + PropertiesQuery is used for all properties for alltypes of nodes like: + classes or collection endpoints, members,object. + """ + def __init__(self): + self.redis_connection = RedisProxy() + self.handle_data = HandleData() + self.connection = self.redis_connection.get_connection() + self._data = self.handle_data + + + def get_classes_properties(self,query): + """ + Show the given type of property of given Class endpoint. + """ + endpoint, query = query.split(" ") + get_data = self.connection.execute_command('GRAPH.QUERY', + 'apidoc','MATCH ( p:classes ) WHERE (p.type="{}") RETURN p.{}'.format(endpoint,query)) + print("class",endpoint,query) + return self._data.show_data(get_data) + + def get_collection_properties(self,query): + """ + Show the given type of property of given collection endpoint. + """ + endpoint, query = query.split(" ") + + get_data = self.connection.execute_command('GRAPH.QUERY', + 'apidoc','MATCH ( p:collection ) WHERE (p.type="{}") RETURN p.{}'.format(endpoint,query)) + + print("collection",endpoint,query) + return self._data.show_data(get_data) + + def get_members_properties(self,query): + """ + Show the given type of property of given member. + """ + endpoint, query = query.split(" ") + get_data = self.connection.execute_command('GRAPH.QUERY', + 'apidoc','MATCH ( p:{} ) RETURN p.id,p.{}'.format(endpoint,query)) + + print("member",endpoint,query) + return self._data.show_data(get_data) + + def get_object_property(self,query): + """ + Show the given type of property of given object. + """ + endpoint,query = query.split(" ") endpoint = endpoint.replace("object","") -# endpoint = "/api/" + endpoint - print("object") - reply = r.execute_command('GRAPH.QUERY', + index = endpoint.find("Collection") + id_ = "object"+endpoint[5:index] + print(id_,endpoint) + get_data = self.connection.execute_command('GRAPH.QUERY', 'apidoc','MATCH ( p:{}) WHERE (p.parent_id = "{}") RETURN p.{}'.format(id_,endpoint,query)) - count = 0 - elif "Collection" in endpoint: - reply = r.execute_command('GRAPH.QUERY', - 'apidoc','MATCH ( p:collection ) WHERE (p.type="{}") RETURN p.{}'.format(endpoint,query)) - count = 2 - print("collection_endpoint") - else: - reply = r.execute_command('GRAPH.QUERY', - 'apidoc','MATCH ( p:classes ) WHERE (p.type="{}") RETURN p.{}'.format(endpoint,query)) - count = 0 - print("class_endpoint") - for objects in reply: - count+=1 - if count%2!=0: - split_data(count,objects) - - -def collection_endpoint_members(endpoint): - collect = CollectionEndpoints(graph.redis_graph,graph.class_endpoints) - r = redis.StrictRedis(host='localhost', port=6379, db=0) - endpoint = endpoint.replace(" members","") -# collect.load_from_server(endpoint,api_doc,url) - query = "objects"+endpoint[:-10] -# print(query) - reply = r.execute_command('GRAPH.QUERY', - 'apidoc','MATCH (p:{}) WHERE (p.type = "{}") RETURN p.id'.format(query,endpoint[:-10])) -# print(reply) - if len(reply[0])==1: - collect.load_from_server(endpoint,api_doc,url) - reply = r.execute_command('GRAPH.QUERY', - 'apidoc','MATCH (p:{}) WHERE (p.type = "{}") RETURN p.id'.format(query,endpoint[:-10])) - count=2 - print("members") - for objects in reply: - count+=1 - if count%2!=0: - split_data(count,objects) - - -#def compare_property(query): -# endpoint_type, query1 = query.split(" ",1) -## query = list(map(str.strip, query.split(' '))) -# r = redis.StrictRedis(host='localhost', port=6379, db=0) -# query = endpoint_type -# endpoint_type = "objects" + endpoint_type -# print(endpoint_type,query1) -# reply = r.execute_command('GRAPH.QUERY', -# 'apidoc','MATCH ( p:{} ) WHERE (p.type= "{}") RETURN p.property_value'.format(endpoint_type,query)) -# count = 0 -# for objects in reply: -# count+=1 -# if count%2!=0: -# for obj in objects: -# string = obj.decode('utf-8') -# print (string) -# map_string = map(str.strip,string.split(',')) -# property_list = list(map_string) -# check = property_list.pop() -# property_list.append(check.replace("\x00","")) -# for query1 in dict(property_list[0]): -# print(query1) -# print (property_list[0]) -# print(reply) + + print("object",endpoint,query) + return self._data.show_data(get_data) + + +class CompareProperties: + """pending + """ + def __init__(self): + """pending""" + + + +class QueryFacades: + """ + It is used for call the above defined functions based on given query. + """ + def __init__(self,api_doc,url): + + self.endpoint_query = EndpointQuery() + self.api_doc = api_doc + self.url = url + self.properties = PropertiesQuery() + + def initialize(self): + """ + Initialize is used to initialize the graph for given url. + """ + print("just initialize") + + graph.main(self.url,self.api_doc) + + def user_query(self,query): + """ + It calls function based on queries type. + """ + if query == "endpoints": + self.endpoint_query.get_allEndpoints(query) + elif query == "classEndpoints": + self.endpoint_query.get_classEndpoints(query) + elif query == "collectionEndpoints": + self.endpoint_query.get_collectionEndpoints(query) + elif "members" in query: + self.members = CollectionmembersQuery(self.api_doc,self.url) + self.members.get_members(query) + elif "objects" in query: + self.properties.get_members_properties(query) + elif "object" in query: + self.properties.get_object_property(query) + elif "Collection" in query: + self.properties.get_collection_properties(query) + else: + self.properties.get_classes_properties(query) def main(): - initialize= 0 - while(1): - if initialize ==0: - print("just initialize") - global url + """ + Take URL as an input and make graph using initilize function. + Querying still user wants or still user enters the exit. + """ + url =input("url>>>") + handle_data = HandleData() + apidoc = handle_data.load_data(url + "/vocab") + while (1): + if apidoc =="error": + print ("enter right url") url =input("url>>>") - apidoc = final_file(url + "/vocab") - global api_doc - api_doc = doc_maker.create_doc(apidoc) - graph.main(url,api_doc) - initialize+=1 - elif initialize >0: -# print("for query write your query in queryformat") - print("for exit enter EXIT or write query in queryformat") - query = input("query>>>") - if query == "EXIT": - break - else: - query = query.replace("show ","") - if query=="endpoint" or query == "collectionEndpoint" or query == "classEndpoint": - endpoints_query(query) - elif "members" in query: - collection_endpoint_members(query) - elif "properties" in query or "operations" in query or "property_value" in query or "type" in query or "id" in query: - operation_and_property_query(query) -# else: -# compare_property(query) + apidoc = handle_data.load_data(url + "/vocab") + else: + break + + api_doc = doc_maker.create_doc(apidoc) + facades = QueryFacades(api_doc,url) + facades.initialize() + global check_list + check_list = [] + while(1): + print("press exit to quit") + query = input(">>>") + if query == "exit": + break + else: + facades.user_query(query) if __name__ =="__main__": From b2c25af4403ccc43d8e24081ba59a107db99d372 Mon Sep 17 00:00:00 2001 From: sandeepsajan0 Date: Wed, 27 Jun 2018 12:18:43 +0530 Subject: [PATCH 07/15] follow style pep8 --- hydra-redis/classes_objects.py | 16 ++- hydra-redis/collections_endpoint.py | 27 ++-- hydra-redis/querying_mechanism.py | 210 ++++++++++++++++------------ 3 files changed, 152 insertions(+), 101 deletions(-) diff --git a/hydra-redis/classes_objects.py b/hydra-redis/classes_objects.py index eeed6c5..0b717e8 100644 --- a/hydra-redis/classes_objects.py +++ b/hydra-redis/classes_objects.py @@ -34,14 +34,17 @@ def get_operation(self, api_doc, endpoint): return str(endpoint_method) - def property_value(self,src_node,member,id_): - id_=str("objects"+id_) + def property_value(self, src_node, member, id_): + id_ = str("objects" + id_) node_properties = {} for members in member: node_properties[members] = str(member[members]) - node = Node(label = "properties",alias = str(id_), properties = node_properties) + node = Node( + label="properties", + alias=str(id_), + properties=node_properties) self.redis_graph.add_node(node) - edge = Edge(src_node,"hasproperty",node) + edge = Edge(src_node, "hasproperty", node) self.redis_graph.add_edge(edge) def objects_property( @@ -71,7 +74,10 @@ def objects_property( node_alias = str(objects_node.alias + str(obj)).lower() # key for the node of the object node_properties["parent_id"] = str(objects_node.properties["@id"]) - object_node = self.addNode(str("object"+str(objects_node.properties["@type"])), node_alias, node_properties) + object_node = self.addNode( + str("object"+str(objects_node.properties["@type"])), + node_alias, + node_properties) self.addEdge(objects_node, "has" + str(obj), object_node) # set edge between the object and its parent object if endpoint_prop: diff --git a/hydra-redis/collections_endpoint.py b/hydra-redis/collections_endpoint.py index 01bab05..2c813bc 100644 --- a/hydra-redis/collections_endpoint.py +++ b/hydra-redis/collections_endpoint.py @@ -26,7 +26,7 @@ def collectionobjects( """Creating nodes for all objects stored in collection.""" print("accesing the collection object like events or drones") if endpoint_list: - clas = ClassEndpoints(self.redis_graph,self.class_endpoints) + clas = ClassEndpoints(self.redis_graph, self.class_endpoints) for endpoint in endpoint_list: node_properties = {} no_endpoint_list = [] @@ -44,7 +44,9 @@ def collectionobjects( member_alias = entrypoint_member # key for the object node is memeber_alias member_id = match_obj.group(3) - print("member alias and id", member_alias.capitalize(), member_id) + print("member alias and url+id", + member_alias.capitalize(), + base_url+member_id) new_url1 = new_url + "/" + member_id new_file1 = self.fetch_data(new_url1) # object data retrieving from the server @@ -79,21 +81,27 @@ def collectionobjects( node_properties["@id"] = str(endpoint["@id"]) node_properties["@type"] = str(endpoint["@type"]) - member[endpoint["@type"]]= str(endpoint["@id"]) + member[endpoint["@type"]] = str(endpoint["@id"]) node_properties["property_value"] = str(member) node_properties["properties"] = str(supported_property_list) collection_object_node = clas.addNode( - str("objects"+str(endpoint["@type"])), str(member_alias.capitalize()), node_properties) + str("objects"+str(endpoint["@type"])), + str(member_alias.capitalize()), + node_properties) # add object as a node in redis - clas.addEdge(endpoint_collection_node, "has_" + - str(endpoint["@type"]), collection_object_node) - + clas.addEdge(endpoint_collection_node, + "has_" + str(endpoint["@type"]), + collection_object_node) + # set an edge between the collection and its object print( "property of endpoint which can be class but not endpoint", no_endpoint_list ) - clas.property_value(collection_object_node,member,member_alias.capitalize()) + clas.property_value( + collection_object_node, + member, + member_alias.capitalize()) if endpoint_property_list: for endpoint_property in endpoint_property_list: for nodes in self.redis_graph.nodes.values(): @@ -122,8 +130,7 @@ def load_from_server( "check url for endpoint", url + "/" + endpoint) - new_url = url + "/"+\ - endpoint + new_url = url + "/" + endpoint # url for every collection endpoint new_file = self.fetch_data(new_url) # retrieving the objects from the collection endpoint diff --git a/hydra-redis/querying_mechanism.py b/hydra-redis/querying_mechanism.py index 686b0a0..a630d98 100644 --- a/hydra-redis/querying_mechanism.py +++ b/hydra-redis/querying_mechanism.py @@ -1,6 +1,5 @@ import redis import hydra_graph as graph -import hydrus import urllib.request import json from hydrus.hydraspec import doc_maker @@ -15,9 +14,10 @@ class HandleData: Work of load_data is to fetch the data from server and Work of show_data is to show the data of Redis in readable format. """ - def load_data(self,url): + + def load_data(self, url): """ - Load the data for the given url and return it. + Load the data for the given url and return it. Also handle with the HTTPError, it prints the error """ try: @@ -31,31 +31,32 @@ def load_data(self,url): else: return json.loads(response.read().decode('utf-8')) - def show_data(self,get_data): + def show_data(self, get_data): """ Make the given data readable, because now it is in binary string form. Count is using for avoid stuffs like query internal execution time. """ count = 0 for objects in get_data: - count+=1 - if count%2!=0: + count += 1 + if count % 2 != 0: for obj in objects: string = obj.decode('utf-8') - map_string = map(str.strip,string.split(',')) + map_string = map(str.strip, string.split(',')) property_list = list(map_string) check = property_list.pop() - property_list.append(check.replace("\x00","")) - print (property_list) + property_list.append(check.replace("\x00", "")) + print(property_list) class RedisProxy: """ RedisProxy is used for make a connection to the Redis. """ + def __init__(self): self.connection = redis.StrictRedis(host='localhost', port=6379, db=0) - + def get_connection(self): return self.connection @@ -64,40 +65,48 @@ class EndpointQuery: """ EndpointQuery is used for get the endpoints from the Redis. """ + def __init__(self): self.redis_connection = RedisProxy() self.handle_data = HandleData() self.connection = self.redis_connection.get_connection() self._data = self.handle_data - def get_allEndpoints(self,query): + def get_allEndpoints(self, query): """ It will return both type(class and collection) of endpoints. """ - get_data = self.connection.execute_command('GRAPH.QUERY','apidoc', "MATCH (p:classes) RETURN p") + \ - self.connection.execute_command ('GRAPH.QUERY','apidoc', "MATCH (p:collection) RETURN p") + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + "MATCH (p:classes) RETURN p") + self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + "MATCH (p:collection) RETURN p") print("classEndpoints + CollectionEndpoints") return self._data.show_data(get_data) - - def get_classEndpoints(self,query): + + def get_classEndpoints(self, query): """ It will return all class Endpoints. """ - get_data = self.connection.execute_command('GRAPH.QUERY','apidoc', "MATCH (p:classes) RETURN p") + get_data = self.connection.execute_command( + 'GRAPH.QUERY', 'apidoc', "MATCH (p:classes) RETURN p") print("classEndpoints") return self._data.show_data(get_data) - - def get_collectionEndpoints(self,query): + + def get_collectionEndpoints(self, query): """ It will returns all collection Endpoints. """ - get_data = self.connection.execute_command ('GRAPH.QUERY','apidoc', "MATCH (p:collection) RETURN p") - + get_data = self.connection.execute_command( + 'GRAPH.QUERY', 'apidoc', "MATCH (p:collection) RETURN p") + print("collectoinEndpoints") - + return self._data.show_data(get_data) @@ -108,7 +117,8 @@ class CollectionmembersQuery: And after that it can query from Redis memory. Check_list is using for track which collection endpoint data is in Redis. """ - def __init__(self,api_doc,url): + + def __init__(self, api_doc, url): self.redis_connection = RedisProxy() self.handle_data = HandleData() self.connection = self.redis_connection.get_connection() @@ -119,34 +129,41 @@ def __init__(self,api_doc,url): self.api_doc = api_doc self.url = url - def data_from_server(self,endpoint): + def data_from_server(self, endpoint): """ Load data from the server for first time. """ - self.collect.load_from_server(endpoint,self.api_doc,self.url) - - get_data = self.connection.execute_command('GRAPH.QUERY', - 'apidoc','MATCH (p:collection) WHERE (p.type = "{}") RETURN p.members'.format(endpoint)) + self.collect.load_from_server(endpoint, self.api_doc, self.url) + + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + 'MATCH(p:collection) WHERE(p.type="{}") RETURN p.members'.format( + endpoint)) return get_data - - def get_members(self,query): + + def get_members(self, query): """ Gets Data from the Redis. """ - endpoint = query.replace(" members","") + endpoint = query.replace(" members", "") print(check_list) if endpoint in check_list: - get_data = self.connection.execute_command('GRAPH.QUERY', - 'apidoc','MATCH (p:collection) WHERE (p.type = "{}") RETURN p.members'.format(endpoint)) + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + """MATCH(p:collection) + WHERE(p.type='{}') + RETURN p.members""".format( + endpoint)) else: check_list.append(endpoint) print(check_list) get_data = self.data_from_server(endpoint) - - print(endpoint," members") - return self._data.show_data(get_data) + print(endpoint, " members") + return self._data.show_data(get_data) class PropertiesQuery: @@ -154,90 +171,108 @@ class PropertiesQuery: PropertiesQuery is used for all properties for alltypes of nodes like: classes or collection endpoints, members,object. """ + def __init__(self): self.redis_connection = RedisProxy() self.handle_data = HandleData() self.connection = self.redis_connection.get_connection() self._data = self.handle_data - - - def get_classes_properties(self,query): + + def get_classes_properties(self, query): """ Show the given type of property of given Class endpoint. """ endpoint, query = query.split(" ") - get_data = self.connection.execute_command('GRAPH.QUERY', - 'apidoc','MATCH ( p:classes ) WHERE (p.type="{}") RETURN p.{}'.format(endpoint,query)) - print("class",endpoint,query) + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + 'MATCH ( p:classes ) WHERE (p.type="{}") RETURN p.{}'.format( + endpoint, + query)) + print("class", endpoint, query) return self._data.show_data(get_data) - def get_collection_properties(self,query): + def get_collection_properties(self, query): """ Show the given type of property of given collection endpoint. """ endpoint, query = query.split(" ") - - get_data = self.connection.execute_command('GRAPH.QUERY', - 'apidoc','MATCH ( p:collection ) WHERE (p.type="{}") RETURN p.{}'.format(endpoint,query)) - - print("collection",endpoint,query) + + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + 'MATCH ( p:collection ) WHERE (p.type="{}") RETURN p.{}'.format( + endpoint, + query)) + + print("collection", endpoint, query) return self._data.show_data(get_data) - def get_members_properties(self,query): + def get_members_properties(self, query): """ Show the given type of property of given member. """ endpoint, query = query.split(" ") - get_data = self.connection.execute_command('GRAPH.QUERY', - 'apidoc','MATCH ( p:{} ) RETURN p.id,p.{}'.format(endpoint,query)) - - print("member",endpoint,query) + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + 'MATCH ( p:{} ) RETURN p.id,p.{}'.format( + endpoint, + query)) + + print("member", endpoint, query) return self._data.show_data(get_data) - def get_object_property(self,query): + def get_object_property(self, query): """ Show the given type of property of given object. """ - endpoint,query = query.split(" ") - endpoint = endpoint.replace("object","") + endpoint, query = query.split(" ") + endpoint = endpoint.replace("object", "") index = endpoint.find("Collection") - id_ = "object"+endpoint[5:index] - print(id_,endpoint) - get_data = self.connection.execute_command('GRAPH.QUERY', - 'apidoc','MATCH ( p:{}) WHERE (p.parent_id = "{}") RETURN p.{}'.format(id_,endpoint,query)) - - print("object",endpoint,query) + id_ = "object" + endpoint[5:index] + print(id_, endpoint) + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + 'MATCH ( p:{}) WHERE (p.parent_id = "{}") RETURN p.{}'.format( + id_, + endpoint, + query)) + + print("object", endpoint, query) return self._data.show_data(get_data) class CompareProperties: - """pending + """pending, I'll update with faceted indexing' """ + def __init__(self): """pending""" - class QueryFacades: """ It is used for call the above defined functions based on given query. """ - def __init__(self,api_doc,url): + + def __init__(self, api_doc, url): self.endpoint_query = EndpointQuery() self.api_doc = api_doc self.url = url self.properties = PropertiesQuery() - + def initialize(self): """ Initialize is used to initialize the graph for given url. """ print("just initialize") - - graph.main(self.url,self.api_doc) - - def user_query(self,query): + + graph.main(self.url, self.api_doc) + + def user_query(self, query): """ It calls function based on queries type. """ @@ -248,7 +283,7 @@ def user_query(self,query): elif query == "collectionEndpoints": self.endpoint_query.get_collectionEndpoints(query) elif "members" in query: - self.members = CollectionmembersQuery(self.api_doc,self.url) + self.members = CollectionmembersQuery(self.api_doc, self.url) self.members.get_members(query) elif "objects" in query: self.properties.get_members_properties(query) @@ -265,19 +300,19 @@ def main(): Take URL as an input and make graph using initilize function. Querying still user wants or still user enters the exit. """ - url =input("url>>>") + url = input("url>>>") handle_data = HandleData() apidoc = handle_data.load_data(url + "/vocab") while (1): - if apidoc =="error": - print ("enter right url") - url =input("url>>>") + if apidoc == "error": + print("enter right url") + url = input("url>>>") apidoc = handle_data.load_data(url + "/vocab") else: break - + api_doc = doc_maker.create_doc(apidoc) - facades = QueryFacades(api_doc,url) + facades = QueryFacades(api_doc, url) facades.initialize() global check_list check_list = [] @@ -290,18 +325,21 @@ def main(): facades.user_query(query) -if __name__ =="__main__": - print ("querying format") - print ("for endpoint:- show endpoint") - print ("for class_endpoint:- show classEndpoint") - print ("for collection_endpoint:- show collectionEndpoint") - print ("for members of collection_endpoint:-show members") - print ("for properties of any member:- show object properties ") - print ("for properties of objects:- show objects properties") - print ("for collection or class properties:- show properties") +if __name__ == "__main__": + print("querying format") + print("for endpoint:- show endpoint") + print("for class_endpoint:- show classEndpoint") + print("for collection_endpoint:- show collectionEndpoint") + print("for members of collection_endpoint:-", + "show members") + print("for properties of any member:-", + "show object properties ") + print("for properties of objects:- show objects properties") + print("for collection or class properties:-", + "show properties") main() # query = input() # query = query.replace("show ","") # operation_and_property_query(query) -## endpoints_query(query) +# endpoints_query(query) # class_endpoint_query("http://35.224.198.158:8080/api","classEndpoint") From ce32264f2f13b61198d97d90ebb260b502651fe8 Mon Sep 17 00:00:00 2001 From: sandeepsajan0 Date: Thu, 28 Jun 2018 12:49:38 +0530 Subject: [PATCH 08/15] add faceted_indexing to properties --- hydra-redis/classes_objects.py | 35 ++++---- hydra-redis/collections_endpoint.py | 36 +++++--- hydra-redis/hydra_graph.py | 13 +-- hydra-redis/querying_mechanism.py | 130 ++++++++++++++++++++++++++-- 4 files changed, 173 insertions(+), 41 deletions(-) diff --git a/hydra-redis/classes_objects.py b/hydra-redis/classes_objects.py index 0b717e8..e3814ce 100644 --- a/hydra-redis/classes_objects.py +++ b/hydra-redis/classes_objects.py @@ -34,19 +34,6 @@ def get_operation(self, api_doc, endpoint): return str(endpoint_method) - def property_value(self, src_node, member, id_): - id_ = str("objects" + id_) - node_properties = {} - for members in member: - node_properties[members] = str(member[members]) - node = Node( - label="properties", - alias=str(id_), - properties=node_properties) - self.redis_graph.add_node(node) - edge = Edge(src_node, "hasproperty", node) - self.redis_graph.add_edge(edge) - def objects_property( self, objects_node, @@ -75,7 +62,7 @@ def objects_property( # key for the node of the object node_properties["parent_id"] = str(objects_node.properties["@id"]) object_node = self.addNode( - str("object"+str(objects_node.properties["@type"])), + str("object" + str(objects_node.properties["@type"])), node_alias, node_properties) self.addEdge(objects_node, "has" + str(obj), object_node) @@ -84,17 +71,27 @@ def objects_property( self.objects_property( object_node, endpoint_prop, api_doc) + def faceted_key(self, key, value): + return ("{}".format("fs:" + key + ":" + value)) + + def faceted_indexing(self, + key, + redis_connection, + member): + for keys in member: + redis_connection.sadd(self.faceted_key(keys, member[keys]), key) + def load_from_server( self, endpoint, api_doc, - base_url): + base_url, + redis_connection): """Loads data from class endpoints like its properties values""" print("check endpoint url....loading data") member = {} endpoint_property = [] - new_url = base_url + \ - endpoint + new_url = base_url + "/" + endpoint # url for the classes endpoint print(new_url) response = urllib.request.urlopen(new_url) @@ -119,8 +116,9 @@ def load_from_server( for node in self.redis_graph.nodes.values(): if node.alias == endpoint: node.properties["property_value"] = str(member) + redis_connection.set((endpoint), member) + self.faceted_indexing(endpoint, redis_connection, member) # update the properties of the node - self.redis_graph.commit() class_object_node = node print(class_object_node) # set edge between the entrypoint and the class endpoint/object @@ -130,6 +128,7 @@ def load_from_server( endpoint_property, no_endpoint_property, api_doc) + self.redis_graph.commit() def endpointclasses( self, diff --git a/hydra-redis/collections_endpoint.py b/hydra-redis/collections_endpoint.py index 2c813bc..a5a0921 100644 --- a/hydra-redis/collections_endpoint.py +++ b/hydra-redis/collections_endpoint.py @@ -16,13 +16,23 @@ def fetch_data(self, new_url): response = urllib.request.urlopen(new_url) return json.loads(response.read().decode('utf-8')) + def faceted_key(self, fs, key, value): + return ("{}".format(fs + ":" + key + ":" + value)) + + def faceted_indexing(self, key, redis_connection, member): + for keys in member: + redis_connection.sadd( + self.faceted_key( + "fs", keys, member[keys]), key) + def collectionobjects( self, endpoint_collection_node, endpoint_list, new_url, api_doc, - url): + url, + redis_connection): """Creating nodes for all objects stored in collection.""" print("accesing the collection object like events or drones") if endpoint_list: @@ -46,7 +56,7 @@ def collectionobjects( member_id = match_obj.group(3) print("member alias and url+id", member_alias.capitalize(), - base_url+member_id) + base_url + member_id) new_url1 = new_url + "/" + member_id new_file1 = self.fetch_data(new_url1) # object data retrieving from the server @@ -75,7 +85,7 @@ def collectionobjects( support_property.title].replace(" ", "")) else: no_endpoint_property = new_file1[ - support_property.title] + support_property.title] else: member[support_property.title] = "null" @@ -83,25 +93,25 @@ def collectionobjects( node_properties["@type"] = str(endpoint["@type"]) member[endpoint["@type"]] = str(endpoint["@id"]) node_properties["property_value"] = str(member) + redis_connection.set((endpoint["@id"]), (member)) +# print(redis_connection.get(endpoint["@id"])) + self.faceted_indexing( + endpoint["@id"], redis_connection, member) node_properties["properties"] = str(supported_property_list) collection_object_node = clas.addNode( - str("objects"+str(endpoint["@type"])), + str("objects" + str(endpoint["@type"])), str(member_alias.capitalize()), node_properties) # add object as a node in redis clas.addEdge(endpoint_collection_node, - "has_" + str(endpoint["@type"]), - collection_object_node) + "has_" + str(endpoint["@type"]), + collection_object_node) # set an edge between the collection and its object print( "property of endpoint which can be class but not endpoint", no_endpoint_list ) - clas.property_value( - collection_object_node, - member, - member_alias.capitalize()) if endpoint_property_list: for endpoint_property in endpoint_property_list: for nodes in self.redis_graph.nodes.values(): @@ -124,7 +134,8 @@ def load_from_server( self, endpoint, api_doc, - url): + url, + redis_connection): """Load data or members from collection endpoint""" print( "check url for endpoint", @@ -147,7 +158,8 @@ def load_from_server( new_file["members"], new_url, api_doc, - url + url, + redis_connection ) self.redis_graph.commit() # for node in self.redis_graph.nodes.values(): diff --git a/hydra-redis/hydra_graph.py b/hydra-redis/hydra_graph.py index cc9a332..a94411f 100644 --- a/hydra-redis/hydra_graph.py +++ b/hydra-redis/hydra_graph.py @@ -51,7 +51,7 @@ def get_apistructure(entrypoint_node, api_doc): url) -def get_endpoints(api_doc): +def get_endpoints(api_doc, redis_connection): """Create node for entrypoint""" print("creating entrypoint node") entrypoint_properties = {} @@ -64,16 +64,18 @@ def get_endpoints(api_doc): alias="Entrypoint", properties=entrypoint_properties) redis_graph.add_node(entrypoint_node) + redis_connection.set("EntryPoint", entrypoint_properties) return get_apistructure(entrypoint_node, api_doc) -def main(new_url,api_doc): + +def main(new_url, api_doc): redis_con = redis.Redis(host='localhost', port=6379) global redis_graph global url - url=new_url + url = new_url redis_graph = Graph("apidoc", redis_con) print("loading... of graph") - get_endpoints(api_doc) + get_endpoints(api_doc, redis_con) print("commiting") redis_graph.commit() # creating whole the graph in redis @@ -89,8 +91,9 @@ def main(new_url,api_doc): # g.view() # #see the graph generated by graphviz + if __name__ == "__main__": url = "http://35.224.198.158:8080/api" apidoc = final_file(url + "/vocab") api_doc = doc_maker.create_doc(apidoc) - main(url,api_doc) + main(url, api_doc) diff --git a/hydra-redis/querying_mechanism.py b/hydra-redis/querying_mechanism.py index a630d98..ba7da1b 100644 --- a/hydra-redis/querying_mechanism.py +++ b/hydra-redis/querying_mechanism.py @@ -5,6 +5,7 @@ from hydrus.hydraspec import doc_maker from urllib.error import URLError, HTTPError from collections_endpoint import CollectionEndpoints +from classes_objects import ClassEndpoints class HandleData: @@ -133,7 +134,10 @@ def data_from_server(self, endpoint): """ Load data from the server for first time. """ - self.collect.load_from_server(endpoint, self.api_doc, self.url) + self.collect.load_from_server(endpoint, + self.api_doc, + self.url, + self.connection) get_data = self.connection.execute_command( 'GRAPH.QUERY', @@ -182,6 +186,7 @@ def get_classes_properties(self, query): """ Show the given type of property of given Class endpoint. """ + query = query.replace("class", "") endpoint, query = query.split(" ") get_data = self.connection.execute_command( 'GRAPH.QUERY', @@ -244,12 +249,116 @@ def get_object_property(self, query): return self._data.show_data(get_data) +class ClassPropertiesValue: + """ + ClassPropertiesValue is used for geting the values for properties of class + And once values get from server and then it stored in Redis. + """ + + def __init__(self, api_doc, url): + self.redis_connection = RedisProxy() + self.handle_data = HandleData() + self.connection = self.redis_connection.get_connection() + self._data = self.handle_data + self.clas = ClassEndpoints(graph.redis_graph, + graph.class_endpoints) + + self.api_doc = api_doc + self.url = url + + def data_from_server(self, endpoint): + """ + Load data from the server for once. + """ + self.clas.load_from_server(endpoint, + self.api_doc, + self.url, + self.connection) + + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + """MATCH(p:classes) + WHERE(p.type='{}') + RETURN p.property_value""".format( + endpoint)) + return get_data + + def get_property_value(self, query): + """ + Load data in Redis if data is not in it with help of checklist. + Checklist have the track on endpoints. + And access the properties values and show them. + """ + query = query.replace("class", "") + endpoint = query.replace(" property_value", "") + print(check_list) + if endpoint in check_list: + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + """MATCH (p:classes) + WHERE (p.type = '{}') + RETURN p.property_value""".format( + endpoint)) + else: + check_list.append(endpoint) + print(check_list) + get_data = self.data_from_server(endpoint) + + print(endpoint, "property_value") + return self._data.show_data(get_data) + + class CompareProperties: - """pending, I'll update with faceted indexing' + """ + CompareProperties is used for extracting endpoints with help of properties + Like input: name Drone1 and model xyz + then output: /api/DroneCollection/2 + with follows objects_property_comparison_list() """ def __init__(self): - """pending""" + self.redis_connection = RedisProxy() + self.handle_data = HandleData() + self.connection = self.redis_connection.get_connection() + self._data = self.handle_data + + def faceted_key(self, key, value): + """ + It is simply concatenate the arguments and make faceted key. + """ + return ("{}".format("fs:" + key + ":" + value)) + + def object_property_comparison_list(self, query): + """ + It takes the argument as a string that can contain many keys and value + And make a list of all keys and values and identify operator(if there) + And execute sinter or sunion commands of Redis over faceted keys. + """ + union = 0 + faceted_list = [] + while(1): + if query.count(" ") > 1: + key, value, query = query.split(" ", 2) + faceted_list.append(self.faceted_key(key, value)) + else: + key, value = query.split(" ") + query = "" + faceted_list.append(self.faceted_key(key, value)) + if len(query) > 0: + operation, query = query.split(" ", 1) + if operation == "or": + union = 1 + + else: + break + if union == 1: + get_data = self.connection.sunion(*faceted_list) + print(get_data) + else: + get_data = self.connection.sinter(*faceted_list) + print(get_data) class QueryFacades: @@ -263,6 +372,7 @@ def __init__(self, api_doc, url): self.api_doc = api_doc self.url = url self.properties = PropertiesQuery() + self.compare = CompareProperties() def initialize(self): """ @@ -276,6 +386,7 @@ def user_query(self, query): """ It calls function based on queries type. """ + query = query.replace("show ", "") if query == "endpoints": self.endpoint_query.get_allEndpoints(query) elif query == "classEndpoints": @@ -291,8 +402,13 @@ def user_query(self, query): self.properties.get_object_property(query) elif "Collection" in query: self.properties.get_collection_properties(query) - else: + elif "class" in query and "property_value" in query: + self.class_property = ClassPropertiesValue(self.api_doc, self.url) + self.class_property.get_property_value(query) + elif "class" in query: self.properties.get_classes_properties(query) + else: + self.compare.object_property_comparison_list(query) def main(): @@ -335,8 +451,10 @@ def main(): print("for properties of any member:-", "show object properties ") print("for properties of objects:- show objects properties") - print("for collection or class properties:-", - "show properties") + print("for collection properties:-", + "show properties") + print("for classes properties:- show class properties") + print("for compare properties:-show and/or ") main() # query = input() # query = query.replace("show ","") From a968bb4f01f7fe42821969dc20f04258fffdbca8 Mon Sep 17 00:00:00 2001 From: sandeepsajan0 Date: Fri, 6 Jul 2018 15:36:11 +0530 Subject: [PATCH 09/15] update querying mechanism with docstrings and comments --- hydra-redis/classes_objects.py | 80 +++++++++---- hydra-redis/collections_endpoint.py | 61 ++++++---- hydra-redis/hydra_graph.py | 5 +- hydra-redis/querying_mechanism.py | 168 ++++++++++++++++++++-------- 4 files changed, 224 insertions(+), 90 deletions(-) diff --git a/hydra-redis/classes_objects.py b/hydra-redis/classes_objects.py index e3814ce..7ef4e18 100644 --- a/hydra-redis/classes_objects.py +++ b/hydra-redis/classes_objects.py @@ -11,18 +11,32 @@ def __init__(self, redis_graph, class_endpoints): self.class_endpoints = class_endpoints def addNode(self, label1, alias1, properties1): - """Add node to the redis graph""" + """ + Add node to the redis graph + :param label1: label for the node. + :param alias1: alias for the node. + :param properties: properties for the node. + :return: Created Node + """ node = Node(label=label1, alias=alias1, properties=properties1) self.redis_graph.add_node(node) return node def addEdge(self, source_node, predicate, dest_node): - """Add edge between nodes in redis graph""" + """Add edge between nodes in redis graph + :param source_node: source node of the edge. + :param predicate: relationship between the source and destination node + :param dest_node: destination node of the edge. + """ edge = Edge(source_node, predicate, dest_node) self.redis_graph.add_edge(edge) def get_operation(self, api_doc, endpoint): - """Return all the supportedOperations for given endpoint""" + """Return all the supportedOperations for given endpoint + :param api_doc: Apidocumentaion for particular url. + :param endpoint: particular endpoint for getting supportedOperations. + :return: All operations for endpoint. + """ endpoint_method = [] for support_operation in api_doc.parsed_classes[ @@ -40,7 +54,12 @@ def objects_property( new_list, no_endpoint_property, api_doc): - """Nodes for every that property which is itself an object""" + """Nodes for every that property which is itself an object + :param objects_node: particular member or class node(parent node). + :param new_list: list of object properties. + :param no_endpoint_property: property_value for new_list properties. + :param api_doc: Apidocumentation of particular url. + """ print("for the property which is an object, should be a node") for obj in new_list: obj = obj.replace("vocab:", "") @@ -57,7 +76,7 @@ def objects_property( endpoint_prop.append(support_property.title) # store all operation and property of object node_properties["properties"] = str(properties_title) - node_properties["property_value"] = str(no_endpoint_property) + node_properties["property_value"] = str(no_endpoint_property[obj]) node_alias = str(objects_node.alias + str(obj)).lower() # key for the node of the object node_properties["parent_id"] = str(objects_node.properties["@id"]) @@ -87,16 +106,23 @@ def load_from_server( api_doc, base_url, redis_connection): - """Loads data from class endpoints like its properties values""" + """Loads data from class endpoints like its properties values + :param endpoint: Particular endpoint for load data from server. + :param api_doc: Apidocumentation for particular url. + :param base_url: Parent url for accessing server. + :param redis_connection: connection for the Redis memory. + """ print("check endpoint url....loading data") member = {} endpoint_property = [] + no_endpoint_property = {} + # new_url is url for the classes endpoint new_url = base_url + "/" + endpoint - # url for the classes endpoint - print(new_url) + # retreiving data for the classes endpoint from server response = urllib.request.urlopen(new_url) new_file = json.loads(response.read().decode('utf-8')) - # retreiving data for the classes endpoint from server + # endpoint_property store all properties which is class/object but not + # endpoint. for support_property in api_doc.parsed_classes[ endpoint][ "class"].supportedProperty: @@ -105,29 +131,37 @@ def load_from_server( if support_property.title in api_doc.parsed_classes: endpoint_property.append(support_property.title) + # members store the properties with its value + # members contains null if supportedProperty has no value in server + # no_endpoint_property store object value of property + # ex: values of state property. if support_property.title in new_file: if isinstance(new_file[support_property.title], str): member[support_property.title] = str( new_file[support_property.title].replace(" ", "")) else: - no_endpoint_property = new_file[support_property.title] + no_endpoint_property[support_property.title] = new_file[ + support_property.title] else: member[support_property.title] = "null" + # Add the property_value in endpoint node properties. for node in self.redis_graph.nodes.values(): if node.alias == endpoint: + # update the properties of the node node.properties["property_value"] = str(member) + # Use faceted index to handle with comparison in properties. redis_connection.set((endpoint), member) self.faceted_indexing(endpoint, redis_connection, member) - # update the properties of the node class_object_node = node print(class_object_node) - # set edge between the entrypoint and the class endpoint/object + # For creating relationship with endpoint_property elements. if endpoint_property: self.objects_property( class_object_node, endpoint_property, no_endpoint_property, api_doc) + # save the graph changes. self.redis_graph.commit() def endpointclasses( @@ -135,36 +169,44 @@ def endpointclasses( entrypoint_node, api_doc, base_url): - """Node for every class which have an endpoint.""" + """Node for every class which have an endpoint. + :param entrypoint_node: Endtrypoint or parent node. + :param api_doc: Apidocumentation for particular url. + :param base_url: parent url for accessing server. + """ print("classes endpoint or accessing classes") endpoint_property_list = {} - # contain all endpoints which have other endpoints as a property. + # endpoint_property_list contain all endpoints + # which have other endpoints as a property ex: State. for endpoint in self.class_endpoints: supported_properties_list = [] - node_properties = {} # node_properties is used for set the properties of node. + node_properties = {} property_list = [] + # store the operations for the endpoint node_properties["operations"] = self.get_operation( api_doc, endpoint) - # store the operations for the endpoint - + # supported_property_list contain all the properties of endpoint. + # property list store the properties which is endpoint as well. for support_property in api_doc.parsed_classes[ endpoint][ "class"].supportedProperty: supported_properties_list.append(support_property.title) + # findout the properties which is also an endpoint. if endpoint != support_property.title: if support_property.title in self.class_endpoints: property_list.append(support_property.title) endpoint_property_list[endpoint] = property_list + # node_properties contains data to store in particular node. node_properties["@id"] = str(self.class_endpoints[endpoint]) node_properties["@type"] = str(endpoint) node_properties["properties"] = str(supported_properties_list) class_object_node = self.addNode( "classes", str(endpoint), node_properties) -# print(class_object_node) + # set edge between the entrypoint and the class endpoint. self.addEdge(entrypoint_node, "has" + endpoint, class_object_node) - # set edge between the entrypoint and the class endpoint/object + # for connect the nodes to endpoint which have endpoint as a property. if endpoint_property_list: for endpoint_property in endpoint_property_list: diff --git a/hydra-redis/collections_endpoint.py b/hydra-redis/collections_endpoint.py index a5a0921..41e3075 100644 --- a/hydra-redis/collections_endpoint.py +++ b/hydra-redis/collections_endpoint.py @@ -12,7 +12,10 @@ def __init__(self, redis_graph, class_endpoints): self.class_endpoints = class_endpoints def fetch_data(self, new_url): - """Fetching data from the server""" + """Fetching data from the server + :param new_url: url for fetching the data. + :return: loaded data. + """ response = urllib.request.urlopen(new_url) return json.loads(response.read().decode('utf-8')) @@ -33,7 +36,14 @@ def collectionobjects( api_doc, url, redis_connection): - """Creating nodes for all objects stored in collection.""" + """Creating nodes for all objects stored in collection. + :param endpoint_collection_node: parent/collection endpoint node. + :param endpoint_list: Members/objects of any collection. + :param new_url: parent url for members/objects + :param api_doc: Apidocumentation for particular url. + :param url: Base url given by user. + :param redis_connection: connection of Redis memory. + """ print("accesing the collection object like events or drones") if endpoint_list: clas = ClassEndpoints(self.redis_graph, self.class_endpoints) @@ -44,6 +54,7 @@ def collectionobjects( member = {} endpoint_property_list = [] supported_property_list = [] + no_endpoint_property = {} match_obj = re.match( r'/(.*)/(.*)/(.*)?', endpoint["@id"], re.M | re.I) base_url = "/{0}/{1}/".format(match_obj.group(1), @@ -54,19 +65,20 @@ def collectionobjects( member_alias = entrypoint_member # key for the object node is memeber_alias member_id = match_obj.group(3) - print("member alias and url+id", - member_alias.capitalize(), - base_url + member_id) - new_url1 = new_url + "/" + member_id - new_file1 = self.fetch_data(new_url1) + member_url = new_url + "/" + member_id # object data retrieving from the server + new_file = self.fetch_data(member_url) for support_operation in api_doc.parsed_classes[ endpoint["@type"] ]["class" ].supportedOperation: endpoint_method.append(support_operation.method) - node_properties["operations"] = str(endpoint_method) # all the operations for the object is stored in method + node_properties["operations"] = str(endpoint_method) + # endpoint_property_list store all properties which is class/object and also an endpoint. + # supported_property_list store all the properties. + # no_endpoint_list store all properties which is class/object + # but not endpoint. for support_property in api_doc.parsed_classes[ endpoint["@type"] ]["class" @@ -78,13 +90,17 @@ def collectionobjects( elif support_property.title in api_doc.parsed_classes: no_endpoint_list.append(support_property.title) - if support_property.title in new_file1: - if isinstance(new_file1[support_property.title], str): + # members contain all the property with value. + # it contains null value for the property which not have value in server. + # no_endpoint_properrty store value for no_endpoint_list. + if support_property.title in new_file: + if isinstance(new_file[support_property.title], str): member[support_property.title] = str( - new_file1[ + new_file[ support_property.title].replace(" ", "")) else: - no_endpoint_property = new_file1[ + no_endpoint_property[ + support_property.title] = new_file[ support_property.title] else: member[support_property.title] = "null" @@ -93,25 +109,21 @@ def collectionobjects( node_properties["@type"] = str(endpoint["@type"]) member[endpoint["@type"]] = str(endpoint["@id"]) node_properties["property_value"] = str(member) + member["type"] = str(endpoint["@type"]) redis_connection.set((endpoint["@id"]), (member)) -# print(redis_connection.get(endpoint["@id"])) self.faceted_indexing( endpoint["@id"], redis_connection, member) node_properties["properties"] = str(supported_property_list) + # add object as a node in redis collection_object_node = clas.addNode( str("objects" + str(endpoint["@type"])), str(member_alias.capitalize()), node_properties) - # add object as a node in redis + # set an edge between the collection and its object clas.addEdge(endpoint_collection_node, "has_" + str(endpoint["@type"]), collection_object_node) - # set an edge between the collection and its object - print( - "property of endpoint which can be class but not endpoint", - no_endpoint_list - ) if endpoint_property_list: for endpoint_property in endpoint_property_list: for nodes in self.redis_graph.nodes.values(): @@ -126,7 +138,7 @@ def collectionobjects( no_endpoint_list, no_endpoint_property, api_doc) -# self.redis_graph.commit() + else: print("NO MEMBERS") @@ -136,7 +148,12 @@ def load_from_server( api_doc, url, redis_connection): - """Load data or members from collection endpoint""" + """Load data or members from collection endpoint + :param endpoint: Given endpoint for load data from server. + :param api_doc: Apidocumentation for particular url. + :param url: Base url given by user. + :param redis_connection: connection to Redis memory. + """ print( "check url for endpoint", url + "/" + @@ -149,9 +166,7 @@ def load_from_server( if node.alias == endpoint: node.properties["members"] = str(new_file["members"]) # update the properties of node by its members -# self.redis_graph.commit() endpoint_collection_node = node -# print(endpoint_collection_node) self.collectionobjects( endpoint_collection_node, diff --git a/hydra-redis/hydra_graph.py b/hydra-redis/hydra_graph.py index a94411f..9d3e50f 100644 --- a/hydra-redis/hydra_graph.py +++ b/hydra-redis/hydra_graph.py @@ -10,7 +10,10 @@ def final_file(url): - """ Open the given url and read and load the Json data.""" + """Open the given url and read and load the Json data. + :param url: given url to access the data from the server. + :return: data loaded from the server. + """ response = urllib.request.urlopen(url) return json.loads(response.read().decode('utf-8')) diff --git a/hydra-redis/querying_mechanism.py b/hydra-redis/querying_mechanism.py index ba7da1b..1bf549f 100644 --- a/hydra-redis/querying_mechanism.py +++ b/hydra-redis/querying_mechanism.py @@ -20,6 +20,8 @@ def load_data(self, url): """ Load the data for the given url and return it. Also handle with the HTTPError, it prints the error + :param url: url for access data from the server. + :return: loaded data """ try: response = urllib.request.urlopen(url) @@ -36,10 +38,14 @@ def show_data(self, get_data): """ Make the given data readable, because now it is in binary string form. Count is using for avoid stuffs like query internal execution time. + :param get_data: data get from the Redis memory. """ count = 0 + all_property_lists = [] for objects in get_data: count += 1 + # Show data only for odd value of count. + # because for even value it contains stuffs like time and etc. if count % 2 != 0: for obj in objects: string = obj.decode('utf-8') @@ -47,7 +53,10 @@ def show_data(self, get_data): property_list = list(map_string) check = property_list.pop() property_list.append(check.replace("\x00", "")) - print(property_list) + if property_list[0] != "NULL": + # print(property_list) + all_property_lists.append(property_list) + return all_property_lists class RedisProxy: @@ -76,6 +85,8 @@ def __init__(self): def get_allEndpoints(self, query): """ It will return both type(class and collection) of endpoints. + :param query: query gets from the user, Ex: endpoints + :returns: data get from the Redis memory. """ get_data = self.connection.execute_command( 'GRAPH.QUERY', @@ -91,6 +102,8 @@ def get_allEndpoints(self, query): def get_classEndpoints(self, query): """ It will return all class Endpoints. + :param query: query get from user, Ex: classEndpoint + :return: get data from the Redis memory. """ get_data = self.connection.execute_command( 'GRAPH.QUERY', 'apidoc', "MATCH (p:classes) RETURN p") @@ -102,6 +115,8 @@ def get_classEndpoints(self, query): def get_collectionEndpoints(self, query): """ It will returns all collection Endpoints. + :param query: query get from the user, Ex: collectionEndpoint + :return: get data from the Redis memory. """ get_data = self.connection.execute_command( 'GRAPH.QUERY', 'apidoc', "MATCH (p:collection) RETURN p") @@ -124,8 +139,8 @@ def __init__(self, api_doc, url): self.handle_data = HandleData() self.connection = self.redis_connection.get_connection() self._data = self.handle_data - self.collect = CollectionEndpoints(graph.redis_graph, - graph.class_endpoints) + self.collection = CollectionEndpoints(graph.redis_graph, + graph.class_endpoints) self.api_doc = api_doc self.url = url @@ -133,25 +148,29 @@ def __init__(self, api_doc, url): def data_from_server(self, endpoint): """ Load data from the server for first time. + :param endpoint: endpoint for getting data from the server. + :return: get data from the Redis memory. """ - self.collect.load_from_server(endpoint, - self.api_doc, - self.url, - self.connection) + self.collection.load_from_server(endpoint, + self.api_doc, + self.url, + self.connection) get_data = self.connection.execute_command( 'GRAPH.QUERY', 'apidoc', 'MATCH(p:collection) WHERE(p.type="{}") RETURN p.members'.format( endpoint)) - return get_data + print(endpoint, " members") + return self._data.show_data(get_data) def get_members(self, query): """ Gets Data from the Redis. + :param query: query get from the user, Ex: DroneCollection members + :return: get data from the Redis memory. """ endpoint = query.replace(" members", "") - print(check_list) if endpoint in check_list: get_data = self.connection.execute_command( 'GRAPH.QUERY', @@ -160,14 +179,13 @@ def get_members(self, query): WHERE(p.type='{}') RETURN p.members""".format( endpoint)) + print(endpoint, " members") + return self._data.show_data(get_data) else: check_list.append(endpoint) print(check_list) - get_data = self.data_from_server(endpoint) - - print(endpoint, " members") - return self._data.show_data(get_data) + return self.data_from_server(endpoint) class PropertiesQuery: @@ -185,6 +203,8 @@ def __init__(self): def get_classes_properties(self, query): """ Show the given type of property of given Class endpoint. + :param query: get query from the user, Ex: classLocation properties + :return: get data from the Redis memory. """ query = query.replace("class", "") endpoint, query = query.split(" ") @@ -200,6 +220,8 @@ def get_classes_properties(self, query): def get_collection_properties(self, query): """ Show the given type of property of given collection endpoint. + :param query: get query from the user, Ex: DroneCollection properties. + :return: get data from the Redis memory. """ endpoint, query = query.split(" ") @@ -216,6 +238,8 @@ def get_collection_properties(self, query): def get_members_properties(self, query): """ Show the given type of property of given member. + :param query: gete query from the user, Ex: objectsDrone properties + :return: get data from the Redis memory. """ endpoint, query = query.split(" ") get_data = self.connection.execute_command( @@ -231,12 +255,13 @@ def get_members_properties(self, query): def get_object_property(self, query): """ Show the given type of property of given object. + :param query: get query from the user,Ex:object 1: key, value, query = query.split(" ", 2) faceted_list.append(self.faceted_key(key, value)) @@ -355,24 +386,37 @@ def object_property_comparison_list(self, query): break if union == 1: get_data = self.connection.sunion(*faceted_list) - print(get_data) + + return self.show_data(get_data) else: get_data = self.connection.sinter(*faceted_list) - print(get_data) + + return self.show_data(get_data) + + def show_data(self, get_data): + """It returns the data in readable format.""" + property_list = [] + for string in get_data: + string1 = string.decode('utf-8') + property_list.append(string1) +# print("list ",property_list) + return property_list class QueryFacades: """ It is used for call the above defined functions based on given query. + Using test as a bool which identify that function is using for tests. """ - def __init__(self, api_doc, url): + def __init__(self, api_doc, url, test): self.endpoint_query = EndpointQuery() self.api_doc = api_doc self.url = url self.properties = PropertiesQuery() self.compare = CompareProperties() + self.test = test def initialize(self): """ @@ -388,60 +432,87 @@ def user_query(self, query): """ query = query.replace("show ", "") if query == "endpoints": - self.endpoint_query.get_allEndpoints(query) + data = self.endpoint_query.get_allEndpoints(query) + return data elif query == "classEndpoints": - self.endpoint_query.get_classEndpoints(query) + data = self.endpoint_query.get_classEndpoints(query) + return data elif query == "collectionEndpoints": - self.endpoint_query.get_collectionEndpoints(query) + data = self.endpoint_query.get_collectionEndpoints(query) + return data elif "members" in query: self.members = CollectionmembersQuery(self.api_doc, self.url) - self.members.get_members(query) + if self.test: + data = self.members.data_from_server( + query.replace(" members", "")) + return data + else: + data = self.members.get_members(query) + return data elif "objects" in query: - self.properties.get_members_properties(query) + data = self.properties.get_members_properties(query) + return data elif "object" in query: - self.properties.get_object_property(query) + data = self.properties.get_object_property(query) + return data elif "Collection" in query: - self.properties.get_collection_properties(query) + data = self.properties.get_collection_properties(query) + return data elif "class" in query and "property_value" in query: self.class_property = ClassPropertiesValue(self.api_doc, self.url) - self.class_property.get_property_value(query) + data = self.class_property.get_property_value(query) + return data elif "class" in query: - self.properties.get_classes_properties(query) + data = self.properties.get_classes_properties(query) + return data else: - self.compare.object_property_comparison_list(query) + data = self.compare.object_property_comparison_list(query) + return data + + +def query(apidoc, url): + """ + It uses only for query purpose. + Querying still user wants or still user enters the exit. + :param apidoc: Apidocumentation for the given url. + :param url: url given by user. + """ + api_doc = doc_maker.create_doc(apidoc) + facades = QueryFacades(api_doc, url, False) + facades.initialize() + global check_list + check_list = [] + while True: + print("press exit to quit") + query = input(">>>") + if query == "exit": + break + elif query == "help": + help() + else: + print(facades.user_query(query)) def main(): """ Take URL as an input and make graph using initilize function. - Querying still user wants or still user enters the exit. + :return: call query function for more query. """ url = input("url>>>") handle_data = HandleData() apidoc = handle_data.load_data(url + "/vocab") - while (1): + while True: if apidoc == "error": print("enter right url") url = input("url>>>") apidoc = handle_data.load_data(url + "/vocab") else: break + return query(apidoc, url) - api_doc = doc_maker.create_doc(apidoc) - facades = QueryFacades(api_doc, url) - facades.initialize() - global check_list - check_list = [] - while(1): - print("press exit to quit") - query = input(">>>") - if query == "exit": - break - else: - facades.user_query(query) - -if __name__ == "__main__": +def help(): + """It prints that how user can query.""" print("querying format") print("for endpoint:- show endpoint") print("for class_endpoint:- show classEndpoint") @@ -450,11 +521,14 @@ def main(): "show members") print("for properties of any member:-", "show object properties ") - print("for properties of objects:- show objects properties") + print("for properties of objects:-show objects properties") print("for collection properties:-", "show properties") print("for classes properties:- show class properties") print("for compare properties:-show and/or ") + + +if __name__ == "__main__": main() # query = input() # query = query.replace("show ","") From 373a8b6767dd5146d54f5f618e26c5459c57d138 Mon Sep 17 00:00:00 2001 From: sandeepsajan0 Date: Fri, 6 Jul 2018 15:38:57 +0530 Subject: [PATCH 10/15] add tests for querying process --- hydra-redis/test/querying_test.py | 127 ++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 hydra-redis/test/querying_test.py diff --git a/hydra-redis/test/querying_test.py b/hydra-redis/test/querying_test.py new file mode 100644 index 0000000..e6a27c0 --- /dev/null +++ b/hydra-redis/test/querying_test.py @@ -0,0 +1,127 @@ +import unittest +import urllib.request +import json +from hydrus.hydraspec import doc_maker +from os import sys, path + +PARENT_DIR = path.dirname(path.dirname(path.abspath(__file__))) +sys.path.append(PARENT_DIR) +import querying_mechanism + + +class Tests: + + def endpointTest(self): + """Test for all the endpoints class + collection endpoints""" + query = "show endpoints" + data = query_facades.user_query(query) + if len(data) == 11: + return True + else: + return False + + def classendpointTest(self): + """Test for class endpoint""" + query = "show classEndpoints" + data = query_facades.user_query(query) + if len(data) == 2: + return True + else: + return False + + def collectionendpointTest(self): + """Test for collection endpoint""" + query = "show collectionEndpoints" + data = query_facades.user_query(query) + if len(data) == 9: + return True + else: + return False + + def CommandCollectionmemberTest(self): + """ + Test for all Commands in CommandCollection. + Data is already stored in check_data from the static data url. + Check_data is used for compare the data retrieve by querying process. + """ + check_data = ['[]'] + query = "show CommandCollection members" + data = query_facades.user_query(query) + if data[1] == check_data: + return True + else: + return False + + def ControllerLogCollectionmemberTest(self): + """ + Test for all controller logs for ControllerLogCollection. + Whole object of ControllerLogCollection is stored in check data. + Check_data is used for compare the data retrieve by querying process. + """ + check_data = [{'@id': '/api/ControllerLogCollection/65', + '@type': 'ControllerLog'}, + {'@id': '/api/ControllerLogCollection/183', + '@type': 'ControllerLog'}, + {'@id': '/api/ControllerLogCollection/374', + '@type': 'ControllerLog'}] + query = "show ControllerLogCollection members" + data = query_facades.user_query(query) + # Make data searchable and comaprable. + data1 = str(data[1]).replace('"', '') + # data retrive from the memory can be distributed: + # like type can be at first position and id can be at second. + # So, check_data split in 3 parts. + # And check all parts are in data retrieve. + if str( + check_data[0]) in data1 and str( + check_data[1]) in data1 and str( + check_data[2]) in data1: + return True + else: + return False + + def DatastreamCollectionmemberTest(self): + """Test for all datastream with Drone ID 2""" + check_data = ['/api/DatastreamCollection/19'] + query = "show DatastreamCollection members" + data = query_facades.user_query(query) + # Here are find the datastream only for those which have DroneID 2. + query = "show DroneID 2 and type Datastream" + data = query_facades.user_query(query) + if data == check_data: + return True + else: + return False + + +class TestQueryingMechanism(unittest.TestCase): + query_test = Tests() + + def test_1_endpoint(self): + self.assertTrue(self.query_test.endpointTest()) + + def test_2_classendpoint(self): + self.assertTrue(self.query_test.classendpointTest()) + + def test_3_collectionendpoint(self): + self.assertTrue(self.query_test.collectionendpointTest()) + + def test_4_CommandCollectionmember(self): + self.assertTrue(self.query_test.CommandCollectionmemberTest()) + + def test_5_ControllerLogCollectionmember(self): + self.assertTrue(self.query_test.ControllerLogCollectionmemberTest()) + + def test_6_DatastreamCollectionmember(self): + self.assertTrue(self.query_test.DatastreamCollectionmemberTest()) + + +if __name__ == "__main__": + url = "https://storage.googleapis.com/api3/api" + vocab_url = url + "/" + "vocab" + response = urllib.request.urlopen(vocab_url) + apidoc = json.loads(response.read().decode('utf-8')) + api_doc = doc_maker.create_doc(apidoc) + query_facades = querying_mechanism.QueryFacades(api_doc, url, True) + query_facades.initialize() + unittest.main() From 6dedee4ab52c9d9c85a0414d87545b2056beeae2 Mon Sep 17 00:00:00 2001 From: sandeepsajan0 Date: Fri, 6 Jul 2018 16:19:59 +0530 Subject: [PATCH 11/15] update coding style --- hydra-redis/hydra_graph.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/hydra-redis/hydra_graph.py b/hydra-redis/hydra_graph.py index 9d3e50f..53bc3d9 100644 --- a/hydra-redis/hydra_graph.py +++ b/hydra-redis/hydra_graph.py @@ -24,28 +24,23 @@ def get_apistructure(entrypoint_node, api_doc): collection_endpoints = {} global class_endpoints class_endpoints = {} - collection = 0 - classes = 0 print("split entrypoint into 2 types of endpoints collection and classes") for support_property in api_doc.entrypoint.entrypoint.supportedProperty: if isinstance( support_property, hydrus.hydraspec.doc_writer.EntryPointClass): class_endpoints[support_property.name] = support_property.id_ - collection = 1 + if isinstance( support_property, hydrus.hydraspec.doc_writer.EntryPointCollection): collection_endpoints[support_property.name] = support_property.id_ - classes = 1 -# print("class_endpoints", class_endpoints) -# print("collection_endpoints", collection_endpoints) - if classes == 1: + if len(class_endpoints.keys())>0: clas = ClassEndpoints(redis_graph, class_endpoints) clas.endpointclasses(entrypoint_node, api_doc, url) - if collection == 1: + if len(collection_endpoints.keys())>0: coll = CollectionEndpoints(redis_graph, class_endpoints) coll.endpointCollection( collection_endpoints, From ffde51eaf5979c94c68fbeb7a727560649e2002c Mon Sep 17 00:00:00 2001 From: sandeepsajan0 Date: Sun, 8 Jul 2018 01:24:10 +0530 Subject: [PATCH 12/15] remove globalvariable, update querying_test --- hydra-redis/hydra_graph.py | 141 +++++++++++++++--------------- hydra-redis/querying_mechanism.py | 34 +++---- hydra-redis/redis_proxy.py | 16 ++++ hydra-redis/test/querying_test.py | 91 ++++++++----------- 4 files changed, 139 insertions(+), 143 deletions(-) create mode 100644 hydra-redis/redis_proxy.py diff --git a/hydra-redis/hydra_graph.py b/hydra-redis/hydra_graph.py index 53bc3d9..0d5dc6d 100644 --- a/hydra-redis/hydra_graph.py +++ b/hydra-redis/hydra_graph.py @@ -8,90 +8,89 @@ from classes_objects import ClassEndpoints from collections_endpoint import CollectionEndpoints +class InitialGraph: + -def final_file(url): - """Open the given url and read and load the Json data. - :param url: given url to access the data from the server. - :return: data loaded from the server. - """ - response = urllib.request.urlopen(url) - return json.loads(response.read().decode('utf-8')) + def final_file(self,url): + """Open the given url and read and load the Json data. + :param url: given url to access the data from the server. + :return: data loaded from the server. + """ + response = urllib.request.urlopen(url) + return json.loads(response.read().decode('utf-8')) -def get_apistructure(entrypoint_node, api_doc): - """ It breaks the endpoint into two parts collection and classes""" - global collection_endpoints - collection_endpoints = {} - global class_endpoints - class_endpoints = {} - print("split entrypoint into 2 types of endpoints collection and classes") - for support_property in api_doc.entrypoint.entrypoint.supportedProperty: - if isinstance( - support_property, - hydrus.hydraspec.doc_writer.EntryPointClass): - class_endpoints[support_property.name] = support_property.id_ + def get_apistructure(self,entrypoint_node, api_doc): + """ It breaks the endpoint into two parts collection and classes""" + self.collection_endpoints = {} + self.class_endpoints = {} + print("split entrypoint into 2 types of endpoints collection and classes") + for support_property in api_doc.entrypoint.entrypoint.supportedProperty: + if isinstance( + support_property, + hydrus.hydraspec.doc_writer.EntryPointClass): + self.class_endpoints[support_property.name] = support_property.id_ - if isinstance( - support_property, - hydrus.hydraspec.doc_writer.EntryPointCollection): - collection_endpoints[support_property.name] = support_property.id_ + if isinstance( + support_property, + hydrus.hydraspec.doc_writer.EntryPointCollection): + self.collection_endpoints[support_property.name] = support_property.id_ - if len(class_endpoints.keys())>0: - clas = ClassEndpoints(redis_graph, class_endpoints) - clas.endpointclasses(entrypoint_node, api_doc, url) + if len(self.class_endpoints.keys())>0: + clas = ClassEndpoints(self.redis_graph, self.class_endpoints) + clas.endpointclasses(entrypoint_node, api_doc, self.url) - if len(collection_endpoints.keys())>0: - coll = CollectionEndpoints(redis_graph, class_endpoints) - coll.endpointCollection( - collection_endpoints, - entrypoint_node, - api_doc, - url) + if len(self.collection_endpoints.keys())>0: + coll = CollectionEndpoints(self.redis_graph, self.class_endpoints) + coll.endpointCollection( + self.collection_endpoints, + entrypoint_node, + api_doc, + self.url) -def get_endpoints(api_doc, redis_connection): - """Create node for entrypoint""" - print("creating entrypoint node") - entrypoint_properties = {} - entrypoint_properties["@id"] = str("vocab:Entrypoint") - entrypoint_properties["url"] = str( - api_doc.entrypoint.url) + str(api_doc.entrypoint.api) - entrypoint_properties["supportedOperation"] = "GET" - entrypoint_node = Node( - label="id", - alias="Entrypoint", - properties=entrypoint_properties) - redis_graph.add_node(entrypoint_node) - redis_connection.set("EntryPoint", entrypoint_properties) - return get_apistructure(entrypoint_node, api_doc) + def get_endpoints(self,api_doc, redis_connection): + """Create node for entrypoint""" + print("creating entrypoint node") + entrypoint_properties = {} + entrypoint_properties["@id"] = str("vocab:Entrypoint") + entrypoint_properties["url"] = str( + api_doc.entrypoint.url) + str(api_doc.entrypoint.api) + entrypoint_properties["supportedOperation"] = "GET" + entrypoint_node = Node( + label="id", + alias="Entrypoint", + properties=entrypoint_properties) + self.redis_graph.add_node(entrypoint_node) + redis_connection.set("EntryPoint", entrypoint_properties) + return self.get_apistructure(entrypoint_node, api_doc) -def main(new_url, api_doc): - redis_con = redis.Redis(host='localhost', port=6379) - global redis_graph - global url - url = new_url - redis_graph = Graph("apidoc", redis_con) - print("loading... of graph") - get_endpoints(api_doc, redis_con) - print("commiting") - redis_graph.commit() - # creating whole the graph in redis - print("done!!!!") - # uncomment below 2 lines for getting nodes for whole graph -# for node in redis_graph.nodes.values(): -# print("\n",node.alias) - # uncomment the below lines for show the graph stored in redis -# g = Digraph('redis_graph', filename='hydra_graph.gv') -# # using graphviz for visualization of graph stored in redis -# for edge in redis_graph.edges: -# g.edge(edge.src_node.alias, edge.dest_node.alias) -# g.view() -# #see the graph generated by graphviz + def main(self,new_url,api_doc): + redis_con = redis.Redis(host='localhost', port=6379) + self.url = new_url + self.redis_graph = Graph("apidoc", redis_con) + print("loading... of graph") + self.get_endpoints(api_doc, redis_con) + print("commiting") + self.redis_graph.commit() + # creating whole the graph in redis + print("done!!!!") + # uncomment below 2 lines for getting nodes for whole graph + # for node in redis_graph.nodes.values(): + # print("\n",node.alias) + # uncomment the below lines for show the graph stored in redis + # g = Digraph('redis_graph', filename='hydra_graph.gv') + # # using graphviz for visualization of graph stored in redis + # for edge in redis_graph.edges: + # g.edge(edge.src_node.alias, edge.dest_node.alias) + # g.view() + # #see the graph generated by graphviz if __name__ == "__main__": url = "http://35.224.198.158:8080/api" apidoc = final_file(url + "/vocab") api_doc = doc_maker.create_doc(apidoc) - main(url, api_doc) + initial_graph = InitialGraph() + initial_graph.main(url, api_doc) diff --git a/hydra-redis/querying_mechanism.py b/hydra-redis/querying_mechanism.py index 1bf549f..30d4f5a 100644 --- a/hydra-redis/querying_mechanism.py +++ b/hydra-redis/querying_mechanism.py @@ -1,11 +1,12 @@ import redis -import hydra_graph as graph +from hydra_graph import InitialGraph import urllib.request import json from hydrus.hydraspec import doc_maker from urllib.error import URLError, HTTPError from collections_endpoint import CollectionEndpoints from classes_objects import ClassEndpoints +from redis_proxy import RedisProxy class HandleData: @@ -46,6 +47,11 @@ def show_data(self, get_data): count += 1 # Show data only for odd value of count. # because for even value it contains stuffs like time and etc. + # ex: Redis provide data like if we query class endpoint + # output like: + # [[endpoints in byte object form],[query execution time:0.5ms]] + # So with the help of count, byte object convert to string + # and also show only useful strings not the query execution time. if count % 2 != 0: for obj in objects: string = obj.decode('utf-8') @@ -59,17 +65,6 @@ def show_data(self, get_data): return all_property_lists -class RedisProxy: - """ - RedisProxy is used for make a connection to the Redis. - """ - - def __init__(self): - self.connection = redis.StrictRedis(host='localhost', port=6379, db=0) - - def get_connection(self): - return self.connection - class EndpointQuery: """ @@ -134,7 +129,7 @@ class CollectionmembersQuery: Check_list is using for track which collection endpoint data is in Redis. """ - def __init__(self, api_doc, url): + def __init__(self, api_doc, url,graph): self.redis_connection = RedisProxy() self.handle_data = HandleData() self.connection = self.redis_connection.get_connection() @@ -280,7 +275,7 @@ class ClassPropertiesValue: And once values get from server and then it stored in Redis. """ - def __init__(self, api_doc, url): + def __init__(self, api_doc, url,graph): self.redis_connection = RedisProxy() self.handle_data = HandleData() self.connection = self.redis_connection.get_connection() @@ -424,7 +419,8 @@ def initialize(self): """ print("just initialize") - graph.main(self.url, self.api_doc) + self.graph = InitialGraph() + self.graph.main(self.url, self.api_doc) def user_query(self, query): """ @@ -441,7 +437,9 @@ def user_query(self, query): data = self.endpoint_query.get_collectionEndpoints(query) return data elif "members" in query: - self.members = CollectionmembersQuery(self.api_doc, self.url) + self.members = CollectionmembersQuery(self.api_doc, + self.url, + self.graph) if self.test: data = self.members.data_from_server( query.replace(" members", "")) @@ -459,7 +457,9 @@ def user_query(self, query): data = self.properties.get_collection_properties(query) return data elif "class" in query and "property_value" in query: - self.class_property = ClassPropertiesValue(self.api_doc, self.url) + self.class_property = ClassPropertiesValue(self.api_doc, + self.url, + self.graph) data = self.class_property.get_property_value(query) return data elif "class" in query: diff --git a/hydra-redis/redis_proxy.py b/hydra-redis/redis_proxy.py new file mode 100644 index 0000000..c345147 --- /dev/null +++ b/hydra-redis/redis_proxy.py @@ -0,0 +1,16 @@ +""" +This is use to provide the connection to Redis memory. +""" + +import redis + +class RedisProxy: + """ + RedisProxy is used for make a connection to the Redis. + """ + + def __init__(self): + self.connection = redis.StrictRedis(host='localhost', port=6379, db=0) + + def get_connection(self): + return self.connection diff --git a/hydra-redis/test/querying_test.py b/hydra-redis/test/querying_test.py index e6a27c0..263b29f 100644 --- a/hydra-redis/test/querying_test.py +++ b/hydra-redis/test/querying_test.py @@ -3,42 +3,50 @@ import json from hydrus.hydraspec import doc_maker from os import sys, path - +#import sys +#sys.path[0] = "/home/sandeep/gsoc_work/python-hydra-agent/hydra-redis" +#import querying_mechanism +#print(sys.path) +#import test +#import hydra_redis PARENT_DIR = path.dirname(path.dirname(path.abspath(__file__))) sys.path.append(PARENT_DIR) import querying_mechanism -class Tests: +class TestQueryingMechanism(unittest.TestCase): - def endpointTest(self): - """Test for all the endpoints class + collection endpoints""" - query = "show endpoints" - data = query_facades.user_query(query) - if len(data) == 11: - return True - else: - return False + def setUp(self): + print("testing start:") - def classendpointTest(self): + def test_1_classendpoint(self): """Test for class endpoint""" + check_data = [['p.id', 'p.operations', 'p.properties', 'p.type'], + ['vocab:EntryPoint/Location', + "['POST'", "'PUT'", "'GET']", + "['Location']", 'Location']] query = "show classEndpoints" data = query_facades.user_query(query) - if len(data) == 2: - return True - else: - return False + self.assertEqual(data,check_data) - def collectionendpointTest(self): + def test_2_collectionendpoint(self): """Test for collection endpoint""" + check_data = ["ControllerLogCollection", + "DroneLogCollection", + "AnomalyCollection", + "DroneCollection", + "CommandCollection", + "HttpApiLogCollection", + "DatastreamCollection", + "MessageCollection"] query = "show collectionEndpoints" data = query_facades.user_query(query) - if len(data) == 9: - return True - else: - return False + for check in check_data: + if check not in str(data): + self.assertTrue(False) + self.assertTrue(True) - def CommandCollectionmemberTest(self): + def test_3_CommandCollectionmember(self): """ Test for all Commands in CommandCollection. Data is already stored in check_data from the static data url. @@ -47,12 +55,9 @@ def CommandCollectionmemberTest(self): check_data = ['[]'] query = "show CommandCollection members" data = query_facades.user_query(query) - if data[1] == check_data: - return True - else: - return False + self.assertEqual(data[1],check_data) - def ControllerLogCollectionmemberTest(self): + def test_4_ControllerLogCollectionmember(self): """ Test for all controller logs for ControllerLogCollection. Whole object of ControllerLogCollection is stored in check data. @@ -76,11 +81,12 @@ def ControllerLogCollectionmemberTest(self): check_data[0]) in data1 and str( check_data[1]) in data1 and str( check_data[2]) in data1: - return True + self.assertTrue(True) else: - return False + self.assertTrue(False) + - def DatastreamCollectionmemberTest(self): + def test_5_DatastreamCollectionmember(self): """Test for all datastream with Drone ID 2""" check_data = ['/api/DatastreamCollection/19'] query = "show DatastreamCollection members" @@ -88,32 +94,7 @@ def DatastreamCollectionmemberTest(self): # Here are find the datastream only for those which have DroneID 2. query = "show DroneID 2 and type Datastream" data = query_facades.user_query(query) - if data == check_data: - return True - else: - return False - - -class TestQueryingMechanism(unittest.TestCase): - query_test = Tests() - - def test_1_endpoint(self): - self.assertTrue(self.query_test.endpointTest()) - - def test_2_classendpoint(self): - self.assertTrue(self.query_test.classendpointTest()) - - def test_3_collectionendpoint(self): - self.assertTrue(self.query_test.collectionendpointTest()) - - def test_4_CommandCollectionmember(self): - self.assertTrue(self.query_test.CommandCollectionmemberTest()) - - def test_5_ControllerLogCollectionmember(self): - self.assertTrue(self.query_test.ControllerLogCollectionmemberTest()) - - def test_6_DatastreamCollectionmember(self): - self.assertTrue(self.query_test.DatastreamCollectionmemberTest()) + self.assertEqual(data,check_data) if __name__ == "__main__": From a18106b39736cfc1a823f052b7e40e2eb11e7a83 Mon Sep 17 00:00:00 2001 From: sandeepsajan0 Date: Sun, 8 Jul 2018 12:45:42 +0530 Subject: [PATCH 13/15] rename directory --- hydra_redis/classes_objects.py | 224 ++++++++++++ hydra_redis/collections_endpoint.py | 211 +++++++++++ hydra_redis/hydra_graph.py | 96 +++++ hydra_redis/querying_mechanism.py | 537 ++++++++++++++++++++++++++++ hydra_redis/redis_proxy.py | 16 + hydra_redis/redis_setup.sh | 24 ++ hydra_redis/test/querying_test.py | 99 +++++ hydra_redis/test/redis_test.py | 114 ++++++ 8 files changed, 1321 insertions(+) create mode 100644 hydra_redis/classes_objects.py create mode 100644 hydra_redis/collections_endpoint.py create mode 100644 hydra_redis/hydra_graph.py create mode 100644 hydra_redis/querying_mechanism.py create mode 100644 hydra_redis/redis_proxy.py create mode 100755 hydra_redis/redis_setup.sh create mode 100644 hydra_redis/test/querying_test.py create mode 100644 hydra_redis/test/redis_test.py diff --git a/hydra_redis/classes_objects.py b/hydra_redis/classes_objects.py new file mode 100644 index 0000000..7ef4e18 --- /dev/null +++ b/hydra_redis/classes_objects.py @@ -0,0 +1,224 @@ +import urllib.request +import json +from redisgraph import Node, Edge + + +class ClassEndpoints: + """Contains all the classes endpoint and the objects""" + + def __init__(self, redis_graph, class_endpoints): + self.redis_graph = redis_graph + self.class_endpoints = class_endpoints + + def addNode(self, label1, alias1, properties1): + """ + Add node to the redis graph + :param label1: label for the node. + :param alias1: alias for the node. + :param properties: properties for the node. + :return: Created Node + """ + node = Node(label=label1, alias=alias1, properties=properties1) + self.redis_graph.add_node(node) + return node + + def addEdge(self, source_node, predicate, dest_node): + """Add edge between nodes in redis graph + :param source_node: source node of the edge. + :param predicate: relationship between the source and destination node + :param dest_node: destination node of the edge. + """ + edge = Edge(source_node, predicate, dest_node) + self.redis_graph.add_edge(edge) + + def get_operation(self, api_doc, endpoint): + """Return all the supportedOperations for given endpoint + :param api_doc: Apidocumentaion for particular url. + :param endpoint: particular endpoint for getting supportedOperations. + :return: All operations for endpoint. + """ + endpoint_method = [] + + for support_operation in api_doc.parsed_classes[ + endpoint][ + "class"].supportedOperation: + endpoint_method.append(support_operation.method) +# print("supportedOperation", endpoint_method) + # all the operations for the object is stored in endpoint_method + + return str(endpoint_method) + + def objects_property( + self, + objects_node, + new_list, + no_endpoint_property, + api_doc): + """Nodes for every that property which is itself an object + :param objects_node: particular member or class node(parent node). + :param new_list: list of object properties. + :param no_endpoint_property: property_value for new_list properties. + :param api_doc: Apidocumentation of particular url. + """ + print("for the property which is an object, should be a node") + for obj in new_list: + obj = obj.replace("vocab:", "") + node_properties = {} + properties_title = [] + endpoint_prop = [] + # node_properties is used for set the properties of node. + node_properties["operations"] = self.get_operation(api_doc, obj) + for support_property in api_doc.parsed_classes[ + obj][ + "class"].supportedProperty: + properties_title.append(support_property.title) + if support_property.title in api_doc.parsed_classes: + endpoint_prop.append(support_property.title) + # store all operation and property of object + node_properties["properties"] = str(properties_title) + node_properties["property_value"] = str(no_endpoint_property[obj]) + node_alias = str(objects_node.alias + str(obj)).lower() + # key for the node of the object + node_properties["parent_id"] = str(objects_node.properties["@id"]) + object_node = self.addNode( + str("object" + str(objects_node.properties["@type"])), + node_alias, + node_properties) + self.addEdge(objects_node, "has" + str(obj), object_node) + # set edge between the object and its parent object + if endpoint_prop: + self.objects_property( + object_node, endpoint_prop, api_doc) + + def faceted_key(self, key, value): + return ("{}".format("fs:" + key + ":" + value)) + + def faceted_indexing(self, + key, + redis_connection, + member): + for keys in member: + redis_connection.sadd(self.faceted_key(keys, member[keys]), key) + + def load_from_server( + self, + endpoint, + api_doc, + base_url, + redis_connection): + """Loads data from class endpoints like its properties values + :param endpoint: Particular endpoint for load data from server. + :param api_doc: Apidocumentation for particular url. + :param base_url: Parent url for accessing server. + :param redis_connection: connection for the Redis memory. + """ + print("check endpoint url....loading data") + member = {} + endpoint_property = [] + no_endpoint_property = {} + # new_url is url for the classes endpoint + new_url = base_url + "/" + endpoint + # retreiving data for the classes endpoint from server + response = urllib.request.urlopen(new_url) + new_file = json.loads(response.read().decode('utf-8')) + # endpoint_property store all properties which is class/object but not + # endpoint. + for support_property in api_doc.parsed_classes[ + endpoint][ + "class"].supportedProperty: + if (endpoint != support_property.title and + support_property.title not in self.class_endpoints): + if support_property.title in api_doc.parsed_classes: + endpoint_property.append(support_property.title) + + # members store the properties with its value + # members contains null if supportedProperty has no value in server + # no_endpoint_property store object value of property + # ex: values of state property. + if support_property.title in new_file: + if isinstance(new_file[support_property.title], str): + member[support_property.title] = str( + new_file[support_property.title].replace(" ", "")) + else: + no_endpoint_property[support_property.title] = new_file[ + support_property.title] + else: + member[support_property.title] = "null" + # Add the property_value in endpoint node properties. + for node in self.redis_graph.nodes.values(): + if node.alias == endpoint: + # update the properties of the node + node.properties["property_value"] = str(member) + # Use faceted index to handle with comparison in properties. + redis_connection.set((endpoint), member) + self.faceted_indexing(endpoint, redis_connection, member) + class_object_node = node + print(class_object_node) + # For creating relationship with endpoint_property elements. + if endpoint_property: + self.objects_property( + class_object_node, + endpoint_property, + no_endpoint_property, + api_doc) + # save the graph changes. + self.redis_graph.commit() + + def endpointclasses( + self, + entrypoint_node, + api_doc, + base_url): + """Node for every class which have an endpoint. + :param entrypoint_node: Endtrypoint or parent node. + :param api_doc: Apidocumentation for particular url. + :param base_url: parent url for accessing server. + """ + print("classes endpoint or accessing classes") + endpoint_property_list = {} + # endpoint_property_list contain all endpoints + # which have other endpoints as a property ex: State. + for endpoint in self.class_endpoints: + supported_properties_list = [] + # node_properties is used for set the properties of node. + node_properties = {} + property_list = [] + # store the operations for the endpoint + node_properties["operations"] = self.get_operation( + api_doc, endpoint) + # supported_property_list contain all the properties of endpoint. + # property list store the properties which is endpoint as well. + for support_property in api_doc.parsed_classes[ + endpoint][ + "class"].supportedProperty: + supported_properties_list.append(support_property.title) + # findout the properties which is also an endpoint. + if endpoint != support_property.title: + if support_property.title in self.class_endpoints: + property_list.append(support_property.title) + + endpoint_property_list[endpoint] = property_list + # node_properties contains data to store in particular node. + node_properties["@id"] = str(self.class_endpoints[endpoint]) + node_properties["@type"] = str(endpoint) + node_properties["properties"] = str(supported_properties_list) + class_object_node = self.addNode( + "classes", str(endpoint), node_properties) + # set edge between the entrypoint and the class endpoint. + self.addEdge(entrypoint_node, "has" + endpoint, class_object_node) + + # for connect the nodes to endpoint which have endpoint as a property. + if endpoint_property_list: + for endpoint_property in endpoint_property_list: + for src_node in self.redis_graph.nodes.values(): + if str(endpoint_property) == src_node.alias: + for endpoints in endpoint_property_list[ + endpoint_property]: + for nodes in self.redis_graph.nodes.values(): + if endpoints == nodes.alias: + self.addEdge( + src_node, + "has_endpoint_property", + nodes) + break + break diff --git a/hydra_redis/collections_endpoint.py b/hydra_redis/collections_endpoint.py new file mode 100644 index 0000000..0db3bc6 --- /dev/null +++ b/hydra_redis/collections_endpoint.py @@ -0,0 +1,211 @@ +import urllib.request +import json +import re +from hydra_redis.classes_objects import ClassEndpoints + + +class CollectionEndpoints: + """Contains all the collections endpoints and objects""" + + def __init__(self, redis_graph, class_endpoints): + self.redis_graph = redis_graph + self.class_endpoints = class_endpoints + + def fetch_data(self, new_url): + """Fetching data from the server + :param new_url: url for fetching the data. + :return: loaded data. + """ + response = urllib.request.urlopen(new_url) + return json.loads(response.read().decode('utf-8')) + + def faceted_key(self, fs, key, value): + return ("{}".format(fs + ":" + key + ":" + value)) + + def faceted_indexing(self, key, redis_connection, member): + for keys in member: + redis_connection.sadd( + self.faceted_key( + "fs", keys, member[keys]), key) + + def collectionobjects( + self, + endpoint_collection_node, + endpoint_list, + new_url, + api_doc, + url, + redis_connection): + """Creating nodes for all objects stored in collection. + :param endpoint_collection_node: parent/collection endpoint node. + :param endpoint_list: Members/objects of any collection. + :param new_url: parent url for members/objects + :param api_doc: Apidocumentation for particular url. + :param url: Base url given by user. + :param redis_connection: connection of Redis memory. + """ + print("accesing the collection object like events or drones") + if endpoint_list: + clas = ClassEndpoints(self.redis_graph, self.class_endpoints) + for endpoint in endpoint_list: + node_properties = {} + no_endpoint_list = [] + endpoint_method = [] + member = {} + endpoint_property_list = [] + supported_property_list = [] + no_endpoint_property = {} + match_obj = re.match( + r'/(.*)/(.*)/(.*)?', endpoint["@id"], re.M | re.I) + base_url = "/{0}/{1}/".format(match_obj.group(1), + match_obj.group(2)) + entrypoint_member = endpoint["@type"].lower( + ) + match_obj.group(3) +# print(base_url, entrypoint_member,endpoint["@type"]) + member_alias = entrypoint_member + # key for the object node is memeber_alias + member_id = match_obj.group(3) + member_url = new_url + "/" + member_id + # object data retrieving from the server + new_file = self.fetch_data(member_url) + for support_operation in api_doc.parsed_classes[ + endpoint["@type"] + ]["class" + ].supportedOperation: + endpoint_method.append(support_operation.method) + # all the operations for the object is stored in method + node_properties["operations"] = str(endpoint_method) + # endpoint_property_list store all properties which is class/object and also an endpoint. + # supported_property_list store all the properties. + # no_endpoint_list store all properties which is class/object + # but not endpoint. + for support_property in api_doc.parsed_classes[ + endpoint["@type"] + ]["class" + ].supportedProperty: + supported_property_list.append(support_property.title) + if support_property.title in self.class_endpoints: + endpoint_property_list.append( + str(support_property.title)) + elif support_property.title in api_doc.parsed_classes: + no_endpoint_list.append(support_property.title) + + # members contain all the property with value. + # it contains null value for the property which not have value in server. + # no_endpoint_properrty store value for no_endpoint_list. + if support_property.title in new_file: + if isinstance(new_file[support_property.title], str): + member[support_property.title] = str( + new_file[ + support_property.title].replace(" ", "")) + else: + no_endpoint_property[ + support_property.title] = new_file[ + support_property.title] + else: + member[support_property.title] = "null" + + node_properties["@id"] = str(endpoint["@id"]) + node_properties["@type"] = str(endpoint["@type"]) + member[endpoint["@type"]] = str(endpoint["@id"]) + node_properties["property_value"] = str(member) + member["type"] = str(endpoint["@type"]) + redis_connection.set((endpoint["@id"]), (member)) + self.faceted_indexing( + endpoint["@id"], redis_connection, member) + node_properties["properties"] = str(supported_property_list) + # add object as a node in redis + collection_object_node = clas.addNode( + str("objects" + str(endpoint["@type"])), + str(member_alias.capitalize()), + node_properties) + # set an edge between the collection and its object + clas.addEdge(endpoint_collection_node, + "has_" + str(endpoint["@type"]), + collection_object_node) + + if endpoint_property_list: + for endpoint_property in endpoint_property_list: + for nodes in self.redis_graph.nodes.values(): + if endpoint_property == nodes.alias: + clas.addEdge( + collection_object_node, + "hasendpointproperty", + nodes) + if no_endpoint_list: + clas.objects_property( + collection_object_node, + no_endpoint_list, + no_endpoint_property, + api_doc) + + else: + print("NO MEMBERS") + + def load_from_server( + self, + endpoint, + api_doc, + url, + redis_connection): + """Load data or members from collection endpoint + :param endpoint: Given endpoint for load data from server. + :param api_doc: Apidocumentation for particular url. + :param url: Base url given by user. + :param redis_connection: connection to Redis memory. + """ + print( + "check url for endpoint", + url + "/" + + endpoint) + new_url = url + "/" + endpoint + # url for every collection endpoint + new_file = self.fetch_data(new_url) + # retrieving the objects from the collection endpoint + for node in self.redis_graph.nodes.values(): + if node.alias == endpoint: + node.properties["members"] = str(new_file["members"]) + # update the properties of node by its members + endpoint_collection_node = node + + self.collectionobjects( + endpoint_collection_node, + new_file["members"], + new_url, + api_doc, + url, + redis_connection + ) + self.redis_graph.commit() +# for node in self.redis_graph.nodes.values(): +# print("\n",node.alias) + + def endpointCollection( + self, + collection_endpoint, + entrypoint_node, + api_doc, + url): + """It makes a node for every collection endpoint.""" + print("accessing every collection in entrypoint") + clas = ClassEndpoints(self.redis_graph, self.class_endpoints) + for endpoint in collection_endpoint: + endpoint_method = [] + node_properties = {} + for support_operation in api_doc.collections[ + endpoint][ + "collection"].supportedOperation: + endpoint_method.append(support_operation.method) + node_properties["operations"] = str(endpoint_method) + # all the operations for the collection endpoint is stored in +# print("supportedOperations",node_properties["operations"]) + node_properties["@id"] = str(collection_endpoint[endpoint]) + node_properties["@type"] = str(endpoint) + endpoint_collection_node = clas.addNode( + "collection", endpoint, node_properties) +# print(endpoint_collection_node) + clas.addEdge( + entrypoint_node, + "has_collection", + endpoint_collection_node) + # set an edge between the entrypoint and collection endpoint diff --git a/hydra_redis/hydra_graph.py b/hydra_redis/hydra_graph.py new file mode 100644 index 0000000..11cd1d8 --- /dev/null +++ b/hydra_redis/hydra_graph.py @@ -0,0 +1,96 @@ +import redis +from redisgraph import Graph, Node +import urllib.request +import json +from hydrus.hydraspec import doc_maker +import hydrus +from graphviz import Digraph +from hydra_redis.classes_objects import ClassEndpoints +from hydra_redis.collections_endpoint import CollectionEndpoints + +class InitialGraph: + + + def final_file(self,url): + """Open the given url and read and load the Json data. + :param url: given url to access the data from the server. + :return: data loaded from the server. + """ + response = urllib.request.urlopen(url) + return json.loads(response.read().decode('utf-8')) + + + def get_apistructure(self,entrypoint_node, api_doc): + """ It breaks the endpoint into two parts collection and classes""" + self.collection_endpoints = {} + self.class_endpoints = {} + print("split entrypoint into 2 types of endpoints collection and classes") + for support_property in api_doc.entrypoint.entrypoint.supportedProperty: + if isinstance( + support_property, + hydrus.hydraspec.doc_writer.EntryPointClass): + self.class_endpoints[support_property.name] = support_property.id_ + + if isinstance( + support_property, + hydrus.hydraspec.doc_writer.EntryPointCollection): + self.collection_endpoints[support_property.name] = support_property.id_ + + if len(self.class_endpoints.keys())>0: + clas = ClassEndpoints(self.redis_graph, self.class_endpoints) + clas.endpointclasses(entrypoint_node, api_doc, self.url) + + if len(self.collection_endpoints.keys())>0: + coll = CollectionEndpoints(self.redis_graph, self.class_endpoints) + coll.endpointCollection( + self.collection_endpoints, + entrypoint_node, + api_doc, + self.url) + + + def get_endpoints(self,api_doc, redis_connection): + """Create node for entrypoint""" + print("creating entrypoint node") + entrypoint_properties = {} + entrypoint_properties["@id"] = str("vocab:Entrypoint") + entrypoint_properties["url"] = str( + api_doc.entrypoint.url) + str(api_doc.entrypoint.api) + entrypoint_properties["supportedOperation"] = "GET" + entrypoint_node = Node( + label="id", + alias="Entrypoint", + properties=entrypoint_properties) + self.redis_graph.add_node(entrypoint_node) + redis_connection.set("EntryPoint", entrypoint_properties) + return self.get_apistructure(entrypoint_node, api_doc) + + + def main(self,new_url,api_doc): + redis_con = redis.Redis(host='localhost', port=6379) + self.url = new_url + self.redis_graph = Graph("apidoc", redis_con) + print("loading... of graph") + self.get_endpoints(api_doc, redis_con) + print("commiting") + self.redis_graph.commit() + # creating whole the graph in redis + print("done!!!!") + # uncomment below 2 lines for getting nodes for whole graph + # for node in redis_graph.nodes.values(): + # print("\n",node.alias) + # uncomment the below lines for show the graph stored in redis + # g = Digraph('redis_graph', filename='hydra_graph.gv') + # # using graphviz for visualization of graph stored in redis + # for edge in redis_graph.edges: + # g.edge(edge.src_node.alias, edge.dest_node.alias) + # g.view() + # #see the graph generated by graphviz + + +if __name__ == "__main__": + url = "http://35.224.198.158:8080/api" + apidoc = final_file(url + "/vocab") + api_doc = doc_maker.create_doc(apidoc) + initial_graph = InitialGraph() + initial_graph.main(url, api_doc) diff --git a/hydra_redis/querying_mechanism.py b/hydra_redis/querying_mechanism.py new file mode 100644 index 0000000..a32a116 --- /dev/null +++ b/hydra_redis/querying_mechanism.py @@ -0,0 +1,537 @@ +import redis +from hydra_redis.hydra_graph import InitialGraph +import urllib.request +import json +from hydrus.hydraspec import doc_maker +from urllib.error import URLError, HTTPError +from hydra_redis.collections_endpoint import CollectionEndpoints +from hydra_redis.classes_objects import ClassEndpoints +from hydra_redis.redis_proxy import RedisProxy + + +class HandleData: + """ + Handle data is used to play with data. + It have two functions load_data and show_data. + Work of load_data is to fetch the data from server and + Work of show_data is to show the data of Redis in readable format. + """ + + def load_data(self, url): + """ + Load the data for the given url and return it. + Also handle with the HTTPError, it prints the error + :param url: url for access data from the server. + :return: loaded data + """ + try: + response = urllib.request.urlopen(url) + except HTTPError as e: + print('Error code: ', e.code) + return ("error") + except URLError as e: + print('Reason: ', e.reason) + return ("error") + else: + return json.loads(response.read().decode('utf-8')) + + def show_data(self, get_data): + """ + Make the given data readable, because now it is in binary string form. + Count is using for avoid stuffs like query internal execution time. + :param get_data: data get from the Redis memory. + """ + count = 0 + all_property_lists = [] + for objects in get_data: + count += 1 + # Show data only for odd value of count. + # because for even value it contains stuffs like time and etc. + # ex: Redis provide data like if we query class endpoint + # output like: + # [[endpoints in byte object form],[query execution time:0.5ms]] + # So with the help of count, byte object convert to string + # and also show only useful strings not the query execution time. + if count % 2 != 0: + for obj in objects: + string = obj.decode('utf-8') + map_string = map(str.strip, string.split(',')) + property_list = list(map_string) + check = property_list.pop() + property_list.append(check.replace("\x00", "")) + if property_list[0] != "NULL": + # print(property_list) + all_property_lists.append(property_list) + return all_property_lists + + + +class EndpointQuery: + """ + EndpointQuery is used for get the endpoints from the Redis. + """ + + def __init__(self): + self.redis_connection = RedisProxy() + self.handle_data = HandleData() + self.connection = self.redis_connection.get_connection() + self._data = self.handle_data + + def get_allEndpoints(self, query): + """ + It will return both type(class and collection) of endpoints. + :param query: query gets from the user, Ex: endpoints + :returns: data get from the Redis memory. + """ + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + "MATCH (p:classes) RETURN p") + self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + "MATCH (p:collection) RETURN p") + print("classEndpoints + CollectionEndpoints") + + return self._data.show_data(get_data) + + def get_classEndpoints(self, query): + """ + It will return all class Endpoints. + :param query: query get from user, Ex: classEndpoint + :return: get data from the Redis memory. + """ + get_data = self.connection.execute_command( + 'GRAPH.QUERY', 'apidoc', "MATCH (p:classes) RETURN p") + + print("classEndpoints") + + return self._data.show_data(get_data) + + def get_collectionEndpoints(self, query): + """ + It will returns all collection Endpoints. + :param query: query get from the user, Ex: collectionEndpoint + :return: get data from the Redis memory. + """ + get_data = self.connection.execute_command( + 'GRAPH.QUERY', 'apidoc', "MATCH (p:collection) RETURN p") + + print("collectoinEndpoints") + + return self._data.show_data(get_data) + + +class CollectionmembersQuery: + """ + CollectionmembersQuery is used for get members of any collectionendpoint. + Once it get the data from the server and store it in Redis. + And after that it can query from Redis memory. + Check_list is using for track which collection endpoint data is in Redis. + """ + + def __init__(self, api_doc, url,graph): + self.redis_connection = RedisProxy() + self.handle_data = HandleData() + self.connection = self.redis_connection.get_connection() + self._data = self.handle_data + self.collection = CollectionEndpoints(graph.redis_graph, + graph.class_endpoints) + + self.api_doc = api_doc + self.url = url + + def data_from_server(self, endpoint): + """ + Load data from the server for first time. + :param endpoint: endpoint for getting data from the server. + :return: get data from the Redis memory. + """ + self.collection.load_from_server(endpoint, + self.api_doc, + self.url, + self.connection) + + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + 'MATCH(p:collection) WHERE(p.type="{}") RETURN p.members'.format( + endpoint)) + print(endpoint, " members") + return self._data.show_data(get_data) + + def get_members(self, query): + """ + Gets Data from the Redis. + :param query: query get from the user, Ex: DroneCollection members + :return: get data from the Redis memory. + """ + endpoint = query.replace(" members", "") + if endpoint in check_list: + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + """MATCH(p:collection) + WHERE(p.type='{}') + RETURN p.members""".format( + endpoint)) + print(endpoint, " members") + return self._data.show_data(get_data) + + else: + check_list.append(endpoint) + print(check_list) + return self.data_from_server(endpoint) + + +class PropertiesQuery: + """ + PropertiesQuery is used for all properties for alltypes of nodes like: + classes or collection endpoints, members,object. + """ + + def __init__(self): + self.redis_connection = RedisProxy() + self.handle_data = HandleData() + self.connection = self.redis_connection.get_connection() + self._data = self.handle_data + + def get_classes_properties(self, query): + """ + Show the given type of property of given Class endpoint. + :param query: get query from the user, Ex: classLocation properties + :return: get data from the Redis memory. + """ + query = query.replace("class", "") + endpoint, query = query.split(" ") + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + 'MATCH ( p:classes ) WHERE (p.type="{}") RETURN p.{}'.format( + endpoint, + query)) + print("class", endpoint, query) + return self._data.show_data(get_data) + + def get_collection_properties(self, query): + """ + Show the given type of property of given collection endpoint. + :param query: get query from the user, Ex: DroneCollection properties. + :return: get data from the Redis memory. + """ + endpoint, query = query.split(" ") + + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + 'MATCH ( p:collection ) WHERE (p.type="{}") RETURN p.{}'.format( + endpoint, + query)) + + print("collection", endpoint, query) + return self._data.show_data(get_data) + + def get_members_properties(self, query): + """ + Show the given type of property of given member. + :param query: gete query from the user, Ex: objectsDrone properties + :return: get data from the Redis memory. + """ + endpoint, query = query.split(" ") + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + 'MATCH ( p:{} ) RETURN p.id,p.{}'.format( + endpoint, + query)) + + print("member", endpoint, query) + return self._data.show_data(get_data) + + def get_object_property(self, query): + """ + Show the given type of property of given object. + :param query: get query from the user,Ex:object 1: + key, value, query = query.split(" ", 2) + faceted_list.append(self.faceted_key(key, value)) + else: + key, value = query.split(" ") + query = "" + faceted_list.append(self.faceted_key(key, value)) + if len(query) > 0: + operation, query = query.split(" ", 1) + if operation == "or": + union = 1 + + else: + break + if union == 1: + get_data = self.connection.sunion(*faceted_list) + + return self.show_data(get_data) + else: + get_data = self.connection.sinter(*faceted_list) + + return self.show_data(get_data) + + def show_data(self, get_data): + """It returns the data in readable format.""" + property_list = [] + for string in get_data: + string1 = string.decode('utf-8') + property_list.append(string1) +# print("list ",property_list) + return property_list + + +class QueryFacades: + """ + It is used for call the above defined functions based on given query. + Using test as a bool which identify that function is using for tests. + """ + + def __init__(self, api_doc, url, test): + + self.endpoint_query = EndpointQuery() + self.api_doc = api_doc + self.url = url + self.properties = PropertiesQuery() + self.compare = CompareProperties() + self.test = test + + def initialize(self): + """ + Initialize is used to initialize the graph for given url. + """ + print("just initialize") + + self.graph = InitialGraph() + self.graph.main(self.url, self.api_doc) + + def user_query(self, query): + """ + It calls function based on queries type. + """ + query = query.replace("show ", "") + if query == "endpoints": + data = self.endpoint_query.get_allEndpoints(query) + return data + elif query == "classEndpoints": + data = self.endpoint_query.get_classEndpoints(query) + return data + elif query == "collectionEndpoints": + data = self.endpoint_query.get_collectionEndpoints(query) + return data + elif "members" in query: + self.members = CollectionmembersQuery(self.api_doc, + self.url, + self.graph) + if self.test: + data = self.members.data_from_server( + query.replace(" members", "")) + return data + else: + data = self.members.get_members(query) + return data + elif "objects" in query: + data = self.properties.get_members_properties(query) + return data + elif "object" in query: + data = self.properties.get_object_property(query) + return data + elif "Collection" in query: + data = self.properties.get_collection_properties(query) + return data + elif "class" in query and "property_value" in query: + self.class_property = ClassPropertiesValue(self.api_doc, + self.url, + self.graph) + data = self.class_property.get_property_value(query) + return data + elif "class" in query: + data = self.properties.get_classes_properties(query) + return data + else: + data = self.compare.object_property_comparison_list(query) + return data + + +def query(apidoc, url): + """ + It uses only for query purpose. + Querying still user wants or still user enters the exit. + :param apidoc: Apidocumentation for the given url. + :param url: url given by user. + """ + api_doc = doc_maker.create_doc(apidoc) + facades = QueryFacades(api_doc, url, False) + facades.initialize() + global check_list + check_list = [] + while True: + print("press exit to quit") + query = input(">>>") + if query == "exit": + break + elif query == "help": + help() + else: + print(facades.user_query(query)) + + +def main(): + """ + Take URL as an input and make graph using initilize function. + :return: call query function for more query. + """ + url = input("url>>>") + handle_data = HandleData() + apidoc = handle_data.load_data(url + "/vocab") + while True: + if apidoc == "error": + print("enter right url") + url = input("url>>>") + apidoc = handle_data.load_data(url + "/vocab") + else: + break + return query(apidoc, url) + + +def help(): + """It prints that how user can query.""" + print("querying format") + print("for endpoint:- show endpoint") + print("for class_endpoint:- show classEndpoint") + print("for collection_endpoint:- show collectionEndpoint") + print("for members of collection_endpoint:-", + "show members") + print("for properties of any member:-", + "show object properties ") + print("for properties of objects:-show objects properties") + print("for collection properties:-", + "show properties") + print("for classes properties:- show class properties") + print("for compare properties:-show and/or ") + + +if __name__ == "__main__": + main() +# query = input() +# query = query.replace("show ","") +# operation_and_property_query(query) +# endpoints_query(query) +# class_endpoint_query("http://35.224.198.158:8080/api","classEndpoint") diff --git a/hydra_redis/redis_proxy.py b/hydra_redis/redis_proxy.py new file mode 100644 index 0000000..c345147 --- /dev/null +++ b/hydra_redis/redis_proxy.py @@ -0,0 +1,16 @@ +""" +This is use to provide the connection to Redis memory. +""" + +import redis + +class RedisProxy: + """ + RedisProxy is used for make a connection to the Redis. + """ + + def __init__(self): + self.connection = redis.StrictRedis(host='localhost', port=6379, db=0) + + def get_connection(self): + return self.connection diff --git a/hydra_redis/redis_setup.sh b/hydra_redis/redis_setup.sh new file mode 100755 index 0000000..cc63c21 --- /dev/null +++ b/hydra_redis/redis_setup.sh @@ -0,0 +1,24 @@ + +#It will remove all the previous docker engine +sudo apt-get remove docker docker-engine docker.io +sudo apt-get update +# Now install docker repo +sudo apt-get install apt-transport-https ca-certificates curl software-properties-common + 2023 curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + +sudo apt-key fingerprint 0EBFCD88 +sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + +# install docker-ce +sudo apt-get update +sudo apt-get install docker-ce + + +# after getting the docker-ce +git clone https://github.com/swilly22/redis-graph.git +cd redis-graph +sudo make docker +# command for run the server +sudo docker run -p 6379:6379 redislabs/redisgraph diff --git a/hydra_redis/test/querying_test.py b/hydra_redis/test/querying_test.py new file mode 100644 index 0000000..92027f1 --- /dev/null +++ b/hydra_redis/test/querying_test.py @@ -0,0 +1,99 @@ +import unittest +import urllib.request +import json +from hydrus.hydraspec import doc_maker +from os import sys, path +from hydra_redis import querying_mechanism + +class TestQueryingMechanism(unittest.TestCase): + + def setUp(self): + print("testing start:") + + def test_1_classendpoint(self): + """Test for class endpoint""" + check_data = [['p.id', 'p.operations', 'p.properties', 'p.type'], + ['vocab:EntryPoint/Location', + "['POST'", "'PUT'", "'GET']", + "['Location']", 'Location']] + query = "show classEndpoints" + data = query_facades.user_query(query) + self.assertEqual(data,check_data) + + def test_2_collectionendpoint(self): + """Test for collection endpoint""" + check_data = ["ControllerLogCollection", + "DroneLogCollection", + "AnomalyCollection", + "DroneCollection", + "CommandCollection", + "HttpApiLogCollection", + "DatastreamCollection", + "MessageCollection"] + query = "show collectionEndpoints" + data = query_facades.user_query(query) + for check in check_data: + if check not in str(data): + self.assertTrue(False) + self.assertTrue(True) + + def test_3_CommandCollectionmember(self): + """ + Test for all Commands in CommandCollection. + Data is already stored in check_data from the static data url. + Check_data is used for compare the data retrieve by querying process. + """ + check_data = ['[]'] + query = "show CommandCollection members" + data = query_facades.user_query(query) + self.assertEqual(data[1],check_data) + + def test_4_ControllerLogCollectionmember(self): + """ + Test for all controller logs for ControllerLogCollection. + Whole object of ControllerLogCollection is stored in check data. + Check_data is used for compare the data retrieve by querying process. + """ + check_data = [{'@id': '/api/ControllerLogCollection/65', + '@type': 'ControllerLog'}, + {'@id': '/api/ControllerLogCollection/183', + '@type': 'ControllerLog'}, + {'@id': '/api/ControllerLogCollection/374', + '@type': 'ControllerLog'}] + query = "show ControllerLogCollection members" + data = query_facades.user_query(query) + # Make data searchable and comaprable. + data1 = str(data[1]).replace('"', '') + # data retrive from the memory can be distributed: + # like type can be at first position and id can be at second. + # So, check_data split in 3 parts. + # And check all parts are in data retrieve. + if str( + check_data[0]) in data1 and str( + check_data[1]) in data1 and str( + check_data[2]) in data1: + self.assertTrue(True) + else: + self.assertTrue(False) + + + def test_5_DatastreamCollectionmember(self): + """Test for all datastream with Drone ID 2""" + check_data = ['/api/DatastreamCollection/19'] + query = "show DatastreamCollection members" + data = query_facades.user_query(query) + # Here are find the datastream only for those which have DroneID 2. + query = "show DroneID 2 and type Datastream" + data = query_facades.user_query(query) + self.assertEqual(data,check_data) + + +if __name__ == "__main__": + url = "https://storage.googleapis.com/api3/api" + vocab_url = url + "/" + "vocab" + response = urllib.request.urlopen(vocab_url) + apidoc = json.loads(response.read().decode('utf-8')) + api_doc = doc_maker.create_doc(apidoc) + query_facades = querying_mechanism.QueryFacades(api_doc, url, True) + query_facades.initialize() + unittest.main() diff --git a/hydra_redis/test/redis_test.py b/hydra_redis/test/redis_test.py new file mode 100644 index 0000000..4a9ae38 --- /dev/null +++ b/hydra_redis/test/redis_test.py @@ -0,0 +1,114 @@ +import unittest +import redis + + +class Tests: + def entry_point(self): + """Test for testing the data stored in entrypoint endpoint""" + + print("entrypoint db=0") + r = redis.StrictRedis(host='localhost', port=6379, db=0) + reply = r.execute_command('GRAPH.QUERY', + 'apidoc', "MATCH (p:id) RETURN p") + property_list = [] + flag = 0 + for objects in reply: + for obj in objects: + if flag == 0: + string = obj.decode('utf-8') + map_string = map(str.strip, string.split(',')) + property_list = list(map_string) + check = property_list.pop() + property_list.append(check.replace("\x00", "")) + print(property_list) + flag += 1 + break + if ("p.id" in property_list and + "p.url" in property_list and + "p.supportedOperation" in property_list): + return True + else: + return False + + def collection_endpoints(self): + """Test for testing the data stored in collection endpoints""" + + print("collection endpoints db=0") + r = redis.StrictRedis(host='localhost', port=6379, db=0) + reply = r.execute_command('GRAPH.QUERY', + 'apidoc', "MATCH (p:collection) RETURN p") + property_list = [] + flag = 0 + for objects in reply: + for obj in objects: + if flag == 0: + string = obj.decode('utf-8') + map_string = map(str.strip, string.split(',')) + property_list = list(map_string) + check = property_list.pop() + property_list.append(check.replace("\x00", "")) + print(property_list) + flag += 1 + break + if ("p.id" in property_list and + "p.operations" in property_list and + "p.members" in property_list): + return True + else: + return False + + def class_endpoints(self): + """Test for testing the data stored in classes endpoints""" + + print("class endpoints db=0") + r = redis.StrictRedis(host='localhost', port=6379, db=0) + reply = r.execute_command('GRAPH.QUERY', + 'apidoc', "MATCH (p:classes) RETURN p") + property_list = [] + flag = 0 + for objects in reply: + for obj in objects: + if flag == 0: + string = obj.decode('utf-8') + map_string = map(str.strip, string.split(',')) + property_list = list(map_string) + check = property_list.pop() + property_list.append(check.replace("\x00", "")) + print(property_list) + flag += 1 + break + if ("p.id" in property_list and + "p.operations" in property_list and + "p.properties" in property_list and + "p.type" in property_list): + return True + else: + return False + + +class TestRedisStructure(unittest.TestCase): + test_redis = Tests() + + @classmethod + def setUpClass(cls): + cls.test_database=redis.StrictRedis(host='localhost', port=6379, db=5) + cls.test_database.set("foo","bar") + cls.test_database.set("hydra","redis") + print("setUpClass db=5 keys:",cls.test_database.keys()) + + def test_entrypoint(self): + self.assertTrue(self.test_redis.entry_point()) + + def test_collectionEndpoints(self): + self.assertTrue(self.test_redis.collection_endpoints()) + + def test_classEndpoints(self): + self.assertTrue(self.test_redis.class_endpoints()) + + @classmethod + def tearDownClass(cls): + cls.test_database.flushdb() + print("tearDownClass db=5 keys:",cls.test_database.get("foo")) + +if __name__ == '__main__': + unittest.main() From 0f189eea3d6e93ec6791da91cd37fd3573194bd9 Mon Sep 17 00:00:00 2001 From: sandeepsajan0 Date: Sun, 8 Jul 2018 12:48:35 +0530 Subject: [PATCH 14/15] remove previous files --- hydra-redis/classes_objects.py | 224 ------------ hydra-redis/collections_endpoint.py | 211 ----------- hydra-redis/hydra_graph.py | 96 ----- hydra-redis/querying_mechanism.py | 537 ---------------------------- hydra-redis/redis_proxy.py | 16 - hydra-redis/redis_setup.sh | 24 -- hydra-redis/test/querying_test.py | 108 ------ hydra-redis/test/redis_test.py | 114 ------ 8 files changed, 1330 deletions(-) delete mode 100644 hydra-redis/classes_objects.py delete mode 100644 hydra-redis/collections_endpoint.py delete mode 100644 hydra-redis/hydra_graph.py delete mode 100644 hydra-redis/querying_mechanism.py delete mode 100644 hydra-redis/redis_proxy.py delete mode 100755 hydra-redis/redis_setup.sh delete mode 100644 hydra-redis/test/querying_test.py delete mode 100644 hydra-redis/test/redis_test.py diff --git a/hydra-redis/classes_objects.py b/hydra-redis/classes_objects.py deleted file mode 100644 index 7ef4e18..0000000 --- a/hydra-redis/classes_objects.py +++ /dev/null @@ -1,224 +0,0 @@ -import urllib.request -import json -from redisgraph import Node, Edge - - -class ClassEndpoints: - """Contains all the classes endpoint and the objects""" - - def __init__(self, redis_graph, class_endpoints): - self.redis_graph = redis_graph - self.class_endpoints = class_endpoints - - def addNode(self, label1, alias1, properties1): - """ - Add node to the redis graph - :param label1: label for the node. - :param alias1: alias for the node. - :param properties: properties for the node. - :return: Created Node - """ - node = Node(label=label1, alias=alias1, properties=properties1) - self.redis_graph.add_node(node) - return node - - def addEdge(self, source_node, predicate, dest_node): - """Add edge between nodes in redis graph - :param source_node: source node of the edge. - :param predicate: relationship between the source and destination node - :param dest_node: destination node of the edge. - """ - edge = Edge(source_node, predicate, dest_node) - self.redis_graph.add_edge(edge) - - def get_operation(self, api_doc, endpoint): - """Return all the supportedOperations for given endpoint - :param api_doc: Apidocumentaion for particular url. - :param endpoint: particular endpoint for getting supportedOperations. - :return: All operations for endpoint. - """ - endpoint_method = [] - - for support_operation in api_doc.parsed_classes[ - endpoint][ - "class"].supportedOperation: - endpoint_method.append(support_operation.method) -# print("supportedOperation", endpoint_method) - # all the operations for the object is stored in endpoint_method - - return str(endpoint_method) - - def objects_property( - self, - objects_node, - new_list, - no_endpoint_property, - api_doc): - """Nodes for every that property which is itself an object - :param objects_node: particular member or class node(parent node). - :param new_list: list of object properties. - :param no_endpoint_property: property_value for new_list properties. - :param api_doc: Apidocumentation of particular url. - """ - print("for the property which is an object, should be a node") - for obj in new_list: - obj = obj.replace("vocab:", "") - node_properties = {} - properties_title = [] - endpoint_prop = [] - # node_properties is used for set the properties of node. - node_properties["operations"] = self.get_operation(api_doc, obj) - for support_property in api_doc.parsed_classes[ - obj][ - "class"].supportedProperty: - properties_title.append(support_property.title) - if support_property.title in api_doc.parsed_classes: - endpoint_prop.append(support_property.title) - # store all operation and property of object - node_properties["properties"] = str(properties_title) - node_properties["property_value"] = str(no_endpoint_property[obj]) - node_alias = str(objects_node.alias + str(obj)).lower() - # key for the node of the object - node_properties["parent_id"] = str(objects_node.properties["@id"]) - object_node = self.addNode( - str("object" + str(objects_node.properties["@type"])), - node_alias, - node_properties) - self.addEdge(objects_node, "has" + str(obj), object_node) - # set edge between the object and its parent object - if endpoint_prop: - self.objects_property( - object_node, endpoint_prop, api_doc) - - def faceted_key(self, key, value): - return ("{}".format("fs:" + key + ":" + value)) - - def faceted_indexing(self, - key, - redis_connection, - member): - for keys in member: - redis_connection.sadd(self.faceted_key(keys, member[keys]), key) - - def load_from_server( - self, - endpoint, - api_doc, - base_url, - redis_connection): - """Loads data from class endpoints like its properties values - :param endpoint: Particular endpoint for load data from server. - :param api_doc: Apidocumentation for particular url. - :param base_url: Parent url for accessing server. - :param redis_connection: connection for the Redis memory. - """ - print("check endpoint url....loading data") - member = {} - endpoint_property = [] - no_endpoint_property = {} - # new_url is url for the classes endpoint - new_url = base_url + "/" + endpoint - # retreiving data for the classes endpoint from server - response = urllib.request.urlopen(new_url) - new_file = json.loads(response.read().decode('utf-8')) - # endpoint_property store all properties which is class/object but not - # endpoint. - for support_property in api_doc.parsed_classes[ - endpoint][ - "class"].supportedProperty: - if (endpoint != support_property.title and - support_property.title not in self.class_endpoints): - if support_property.title in api_doc.parsed_classes: - endpoint_property.append(support_property.title) - - # members store the properties with its value - # members contains null if supportedProperty has no value in server - # no_endpoint_property store object value of property - # ex: values of state property. - if support_property.title in new_file: - if isinstance(new_file[support_property.title], str): - member[support_property.title] = str( - new_file[support_property.title].replace(" ", "")) - else: - no_endpoint_property[support_property.title] = new_file[ - support_property.title] - else: - member[support_property.title] = "null" - # Add the property_value in endpoint node properties. - for node in self.redis_graph.nodes.values(): - if node.alias == endpoint: - # update the properties of the node - node.properties["property_value"] = str(member) - # Use faceted index to handle with comparison in properties. - redis_connection.set((endpoint), member) - self.faceted_indexing(endpoint, redis_connection, member) - class_object_node = node - print(class_object_node) - # For creating relationship with endpoint_property elements. - if endpoint_property: - self.objects_property( - class_object_node, - endpoint_property, - no_endpoint_property, - api_doc) - # save the graph changes. - self.redis_graph.commit() - - def endpointclasses( - self, - entrypoint_node, - api_doc, - base_url): - """Node for every class which have an endpoint. - :param entrypoint_node: Endtrypoint or parent node. - :param api_doc: Apidocumentation for particular url. - :param base_url: parent url for accessing server. - """ - print("classes endpoint or accessing classes") - endpoint_property_list = {} - # endpoint_property_list contain all endpoints - # which have other endpoints as a property ex: State. - for endpoint in self.class_endpoints: - supported_properties_list = [] - # node_properties is used for set the properties of node. - node_properties = {} - property_list = [] - # store the operations for the endpoint - node_properties["operations"] = self.get_operation( - api_doc, endpoint) - # supported_property_list contain all the properties of endpoint. - # property list store the properties which is endpoint as well. - for support_property in api_doc.parsed_classes[ - endpoint][ - "class"].supportedProperty: - supported_properties_list.append(support_property.title) - # findout the properties which is also an endpoint. - if endpoint != support_property.title: - if support_property.title in self.class_endpoints: - property_list.append(support_property.title) - - endpoint_property_list[endpoint] = property_list - # node_properties contains data to store in particular node. - node_properties["@id"] = str(self.class_endpoints[endpoint]) - node_properties["@type"] = str(endpoint) - node_properties["properties"] = str(supported_properties_list) - class_object_node = self.addNode( - "classes", str(endpoint), node_properties) - # set edge between the entrypoint and the class endpoint. - self.addEdge(entrypoint_node, "has" + endpoint, class_object_node) - - # for connect the nodes to endpoint which have endpoint as a property. - if endpoint_property_list: - for endpoint_property in endpoint_property_list: - for src_node in self.redis_graph.nodes.values(): - if str(endpoint_property) == src_node.alias: - for endpoints in endpoint_property_list[ - endpoint_property]: - for nodes in self.redis_graph.nodes.values(): - if endpoints == nodes.alias: - self.addEdge( - src_node, - "has_endpoint_property", - nodes) - break - break diff --git a/hydra-redis/collections_endpoint.py b/hydra-redis/collections_endpoint.py deleted file mode 100644 index 41e3075..0000000 --- a/hydra-redis/collections_endpoint.py +++ /dev/null @@ -1,211 +0,0 @@ -import urllib.request -import json -import re -from classes_objects import ClassEndpoints - - -class CollectionEndpoints: - """Contains all the collections endpoints and objects""" - - def __init__(self, redis_graph, class_endpoints): - self.redis_graph = redis_graph - self.class_endpoints = class_endpoints - - def fetch_data(self, new_url): - """Fetching data from the server - :param new_url: url for fetching the data. - :return: loaded data. - """ - response = urllib.request.urlopen(new_url) - return json.loads(response.read().decode('utf-8')) - - def faceted_key(self, fs, key, value): - return ("{}".format(fs + ":" + key + ":" + value)) - - def faceted_indexing(self, key, redis_connection, member): - for keys in member: - redis_connection.sadd( - self.faceted_key( - "fs", keys, member[keys]), key) - - def collectionobjects( - self, - endpoint_collection_node, - endpoint_list, - new_url, - api_doc, - url, - redis_connection): - """Creating nodes for all objects stored in collection. - :param endpoint_collection_node: parent/collection endpoint node. - :param endpoint_list: Members/objects of any collection. - :param new_url: parent url for members/objects - :param api_doc: Apidocumentation for particular url. - :param url: Base url given by user. - :param redis_connection: connection of Redis memory. - """ - print("accesing the collection object like events or drones") - if endpoint_list: - clas = ClassEndpoints(self.redis_graph, self.class_endpoints) - for endpoint in endpoint_list: - node_properties = {} - no_endpoint_list = [] - endpoint_method = [] - member = {} - endpoint_property_list = [] - supported_property_list = [] - no_endpoint_property = {} - match_obj = re.match( - r'/(.*)/(.*)/(.*)?', endpoint["@id"], re.M | re.I) - base_url = "/{0}/{1}/".format(match_obj.group(1), - match_obj.group(2)) - entrypoint_member = endpoint["@type"].lower( - ) + match_obj.group(3) -# print(base_url, entrypoint_member,endpoint["@type"]) - member_alias = entrypoint_member - # key for the object node is memeber_alias - member_id = match_obj.group(3) - member_url = new_url + "/" + member_id - # object data retrieving from the server - new_file = self.fetch_data(member_url) - for support_operation in api_doc.parsed_classes[ - endpoint["@type"] - ]["class" - ].supportedOperation: - endpoint_method.append(support_operation.method) - # all the operations for the object is stored in method - node_properties["operations"] = str(endpoint_method) - # endpoint_property_list store all properties which is class/object and also an endpoint. - # supported_property_list store all the properties. - # no_endpoint_list store all properties which is class/object - # but not endpoint. - for support_property in api_doc.parsed_classes[ - endpoint["@type"] - ]["class" - ].supportedProperty: - supported_property_list.append(support_property.title) - if support_property.title in self.class_endpoints: - endpoint_property_list.append( - str(support_property.title)) - elif support_property.title in api_doc.parsed_classes: - no_endpoint_list.append(support_property.title) - - # members contain all the property with value. - # it contains null value for the property which not have value in server. - # no_endpoint_properrty store value for no_endpoint_list. - if support_property.title in new_file: - if isinstance(new_file[support_property.title], str): - member[support_property.title] = str( - new_file[ - support_property.title].replace(" ", "")) - else: - no_endpoint_property[ - support_property.title] = new_file[ - support_property.title] - else: - member[support_property.title] = "null" - - node_properties["@id"] = str(endpoint["@id"]) - node_properties["@type"] = str(endpoint["@type"]) - member[endpoint["@type"]] = str(endpoint["@id"]) - node_properties["property_value"] = str(member) - member["type"] = str(endpoint["@type"]) - redis_connection.set((endpoint["@id"]), (member)) - self.faceted_indexing( - endpoint["@id"], redis_connection, member) - node_properties["properties"] = str(supported_property_list) - # add object as a node in redis - collection_object_node = clas.addNode( - str("objects" + str(endpoint["@type"])), - str(member_alias.capitalize()), - node_properties) - # set an edge between the collection and its object - clas.addEdge(endpoint_collection_node, - "has_" + str(endpoint["@type"]), - collection_object_node) - - if endpoint_property_list: - for endpoint_property in endpoint_property_list: - for nodes in self.redis_graph.nodes.values(): - if endpoint_property == nodes.alias: - clas.addEdge( - collection_object_node, - "hasendpointproperty", - nodes) - if no_endpoint_list: - clas.objects_property( - collection_object_node, - no_endpoint_list, - no_endpoint_property, - api_doc) - - else: - print("NO MEMBERS") - - def load_from_server( - self, - endpoint, - api_doc, - url, - redis_connection): - """Load data or members from collection endpoint - :param endpoint: Given endpoint for load data from server. - :param api_doc: Apidocumentation for particular url. - :param url: Base url given by user. - :param redis_connection: connection to Redis memory. - """ - print( - "check url for endpoint", - url + "/" + - endpoint) - new_url = url + "/" + endpoint - # url for every collection endpoint - new_file = self.fetch_data(new_url) - # retrieving the objects from the collection endpoint - for node in self.redis_graph.nodes.values(): - if node.alias == endpoint: - node.properties["members"] = str(new_file["members"]) - # update the properties of node by its members - endpoint_collection_node = node - - self.collectionobjects( - endpoint_collection_node, - new_file["members"], - new_url, - api_doc, - url, - redis_connection - ) - self.redis_graph.commit() -# for node in self.redis_graph.nodes.values(): -# print("\n",node.alias) - - def endpointCollection( - self, - collection_endpoint, - entrypoint_node, - api_doc, - url): - """It makes a node for every collection endpoint.""" - print("accessing every collection in entrypoint") - clas = ClassEndpoints(self.redis_graph, self.class_endpoints) - for endpoint in collection_endpoint: - endpoint_method = [] - node_properties = {} - for support_operation in api_doc.collections[ - endpoint][ - "collection"].supportedOperation: - endpoint_method.append(support_operation.method) - node_properties["operations"] = str(endpoint_method) - # all the operations for the collection endpoint is stored in -# print("supportedOperations",node_properties["operations"]) - node_properties["@id"] = str(collection_endpoint[endpoint]) - node_properties["@type"] = str(endpoint) - endpoint_collection_node = clas.addNode( - "collection", endpoint, node_properties) -# print(endpoint_collection_node) - clas.addEdge( - entrypoint_node, - "has_collection", - endpoint_collection_node) - # set an edge between the entrypoint and collection endpoint diff --git a/hydra-redis/hydra_graph.py b/hydra-redis/hydra_graph.py deleted file mode 100644 index 0d5dc6d..0000000 --- a/hydra-redis/hydra_graph.py +++ /dev/null @@ -1,96 +0,0 @@ -import redis -from redisgraph import Graph, Node -import urllib.request -import json -from hydrus.hydraspec import doc_maker -import hydrus -from graphviz import Digraph -from classes_objects import ClassEndpoints -from collections_endpoint import CollectionEndpoints - -class InitialGraph: - - - def final_file(self,url): - """Open the given url and read and load the Json data. - :param url: given url to access the data from the server. - :return: data loaded from the server. - """ - response = urllib.request.urlopen(url) - return json.loads(response.read().decode('utf-8')) - - - def get_apistructure(self,entrypoint_node, api_doc): - """ It breaks the endpoint into two parts collection and classes""" - self.collection_endpoints = {} - self.class_endpoints = {} - print("split entrypoint into 2 types of endpoints collection and classes") - for support_property in api_doc.entrypoint.entrypoint.supportedProperty: - if isinstance( - support_property, - hydrus.hydraspec.doc_writer.EntryPointClass): - self.class_endpoints[support_property.name] = support_property.id_ - - if isinstance( - support_property, - hydrus.hydraspec.doc_writer.EntryPointCollection): - self.collection_endpoints[support_property.name] = support_property.id_ - - if len(self.class_endpoints.keys())>0: - clas = ClassEndpoints(self.redis_graph, self.class_endpoints) - clas.endpointclasses(entrypoint_node, api_doc, self.url) - - if len(self.collection_endpoints.keys())>0: - coll = CollectionEndpoints(self.redis_graph, self.class_endpoints) - coll.endpointCollection( - self.collection_endpoints, - entrypoint_node, - api_doc, - self.url) - - - def get_endpoints(self,api_doc, redis_connection): - """Create node for entrypoint""" - print("creating entrypoint node") - entrypoint_properties = {} - entrypoint_properties["@id"] = str("vocab:Entrypoint") - entrypoint_properties["url"] = str( - api_doc.entrypoint.url) + str(api_doc.entrypoint.api) - entrypoint_properties["supportedOperation"] = "GET" - entrypoint_node = Node( - label="id", - alias="Entrypoint", - properties=entrypoint_properties) - self.redis_graph.add_node(entrypoint_node) - redis_connection.set("EntryPoint", entrypoint_properties) - return self.get_apistructure(entrypoint_node, api_doc) - - - def main(self,new_url,api_doc): - redis_con = redis.Redis(host='localhost', port=6379) - self.url = new_url - self.redis_graph = Graph("apidoc", redis_con) - print("loading... of graph") - self.get_endpoints(api_doc, redis_con) - print("commiting") - self.redis_graph.commit() - # creating whole the graph in redis - print("done!!!!") - # uncomment below 2 lines for getting nodes for whole graph - # for node in redis_graph.nodes.values(): - # print("\n",node.alias) - # uncomment the below lines for show the graph stored in redis - # g = Digraph('redis_graph', filename='hydra_graph.gv') - # # using graphviz for visualization of graph stored in redis - # for edge in redis_graph.edges: - # g.edge(edge.src_node.alias, edge.dest_node.alias) - # g.view() - # #see the graph generated by graphviz - - -if __name__ == "__main__": - url = "http://35.224.198.158:8080/api" - apidoc = final_file(url + "/vocab") - api_doc = doc_maker.create_doc(apidoc) - initial_graph = InitialGraph() - initial_graph.main(url, api_doc) diff --git a/hydra-redis/querying_mechanism.py b/hydra-redis/querying_mechanism.py deleted file mode 100644 index 30d4f5a..0000000 --- a/hydra-redis/querying_mechanism.py +++ /dev/null @@ -1,537 +0,0 @@ -import redis -from hydra_graph import InitialGraph -import urllib.request -import json -from hydrus.hydraspec import doc_maker -from urllib.error import URLError, HTTPError -from collections_endpoint import CollectionEndpoints -from classes_objects import ClassEndpoints -from redis_proxy import RedisProxy - - -class HandleData: - """ - Handle data is used to play with data. - It have two functions load_data and show_data. - Work of load_data is to fetch the data from server and - Work of show_data is to show the data of Redis in readable format. - """ - - def load_data(self, url): - """ - Load the data for the given url and return it. - Also handle with the HTTPError, it prints the error - :param url: url for access data from the server. - :return: loaded data - """ - try: - response = urllib.request.urlopen(url) - except HTTPError as e: - print('Error code: ', e.code) - return ("error") - except URLError as e: - print('Reason: ', e.reason) - return ("error") - else: - return json.loads(response.read().decode('utf-8')) - - def show_data(self, get_data): - """ - Make the given data readable, because now it is in binary string form. - Count is using for avoid stuffs like query internal execution time. - :param get_data: data get from the Redis memory. - """ - count = 0 - all_property_lists = [] - for objects in get_data: - count += 1 - # Show data only for odd value of count. - # because for even value it contains stuffs like time and etc. - # ex: Redis provide data like if we query class endpoint - # output like: - # [[endpoints in byte object form],[query execution time:0.5ms]] - # So with the help of count, byte object convert to string - # and also show only useful strings not the query execution time. - if count % 2 != 0: - for obj in objects: - string = obj.decode('utf-8') - map_string = map(str.strip, string.split(',')) - property_list = list(map_string) - check = property_list.pop() - property_list.append(check.replace("\x00", "")) - if property_list[0] != "NULL": - # print(property_list) - all_property_lists.append(property_list) - return all_property_lists - - - -class EndpointQuery: - """ - EndpointQuery is used for get the endpoints from the Redis. - """ - - def __init__(self): - self.redis_connection = RedisProxy() - self.handle_data = HandleData() - self.connection = self.redis_connection.get_connection() - self._data = self.handle_data - - def get_allEndpoints(self, query): - """ - It will return both type(class and collection) of endpoints. - :param query: query gets from the user, Ex: endpoints - :returns: data get from the Redis memory. - """ - get_data = self.connection.execute_command( - 'GRAPH.QUERY', - 'apidoc', - "MATCH (p:classes) RETURN p") + self.connection.execute_command( - 'GRAPH.QUERY', - 'apidoc', - "MATCH (p:collection) RETURN p") - print("classEndpoints + CollectionEndpoints") - - return self._data.show_data(get_data) - - def get_classEndpoints(self, query): - """ - It will return all class Endpoints. - :param query: query get from user, Ex: classEndpoint - :return: get data from the Redis memory. - """ - get_data = self.connection.execute_command( - 'GRAPH.QUERY', 'apidoc', "MATCH (p:classes) RETURN p") - - print("classEndpoints") - - return self._data.show_data(get_data) - - def get_collectionEndpoints(self, query): - """ - It will returns all collection Endpoints. - :param query: query get from the user, Ex: collectionEndpoint - :return: get data from the Redis memory. - """ - get_data = self.connection.execute_command( - 'GRAPH.QUERY', 'apidoc', "MATCH (p:collection) RETURN p") - - print("collectoinEndpoints") - - return self._data.show_data(get_data) - - -class CollectionmembersQuery: - """ - CollectionmembersQuery is used for get members of any collectionendpoint. - Once it get the data from the server and store it in Redis. - And after that it can query from Redis memory. - Check_list is using for track which collection endpoint data is in Redis. - """ - - def __init__(self, api_doc, url,graph): - self.redis_connection = RedisProxy() - self.handle_data = HandleData() - self.connection = self.redis_connection.get_connection() - self._data = self.handle_data - self.collection = CollectionEndpoints(graph.redis_graph, - graph.class_endpoints) - - self.api_doc = api_doc - self.url = url - - def data_from_server(self, endpoint): - """ - Load data from the server for first time. - :param endpoint: endpoint for getting data from the server. - :return: get data from the Redis memory. - """ - self.collection.load_from_server(endpoint, - self.api_doc, - self.url, - self.connection) - - get_data = self.connection.execute_command( - 'GRAPH.QUERY', - 'apidoc', - 'MATCH(p:collection) WHERE(p.type="{}") RETURN p.members'.format( - endpoint)) - print(endpoint, " members") - return self._data.show_data(get_data) - - def get_members(self, query): - """ - Gets Data from the Redis. - :param query: query get from the user, Ex: DroneCollection members - :return: get data from the Redis memory. - """ - endpoint = query.replace(" members", "") - if endpoint in check_list: - get_data = self.connection.execute_command( - 'GRAPH.QUERY', - 'apidoc', - """MATCH(p:collection) - WHERE(p.type='{}') - RETURN p.members""".format( - endpoint)) - print(endpoint, " members") - return self._data.show_data(get_data) - - else: - check_list.append(endpoint) - print(check_list) - return self.data_from_server(endpoint) - - -class PropertiesQuery: - """ - PropertiesQuery is used for all properties for alltypes of nodes like: - classes or collection endpoints, members,object. - """ - - def __init__(self): - self.redis_connection = RedisProxy() - self.handle_data = HandleData() - self.connection = self.redis_connection.get_connection() - self._data = self.handle_data - - def get_classes_properties(self, query): - """ - Show the given type of property of given Class endpoint. - :param query: get query from the user, Ex: classLocation properties - :return: get data from the Redis memory. - """ - query = query.replace("class", "") - endpoint, query = query.split(" ") - get_data = self.connection.execute_command( - 'GRAPH.QUERY', - 'apidoc', - 'MATCH ( p:classes ) WHERE (p.type="{}") RETURN p.{}'.format( - endpoint, - query)) - print("class", endpoint, query) - return self._data.show_data(get_data) - - def get_collection_properties(self, query): - """ - Show the given type of property of given collection endpoint. - :param query: get query from the user, Ex: DroneCollection properties. - :return: get data from the Redis memory. - """ - endpoint, query = query.split(" ") - - get_data = self.connection.execute_command( - 'GRAPH.QUERY', - 'apidoc', - 'MATCH ( p:collection ) WHERE (p.type="{}") RETURN p.{}'.format( - endpoint, - query)) - - print("collection", endpoint, query) - return self._data.show_data(get_data) - - def get_members_properties(self, query): - """ - Show the given type of property of given member. - :param query: gete query from the user, Ex: objectsDrone properties - :return: get data from the Redis memory. - """ - endpoint, query = query.split(" ") - get_data = self.connection.execute_command( - 'GRAPH.QUERY', - 'apidoc', - 'MATCH ( p:{} ) RETURN p.id,p.{}'.format( - endpoint, - query)) - - print("member", endpoint, query) - return self._data.show_data(get_data) - - def get_object_property(self, query): - """ - Show the given type of property of given object. - :param query: get query from the user,Ex:object 1: - key, value, query = query.split(" ", 2) - faceted_list.append(self.faceted_key(key, value)) - else: - key, value = query.split(" ") - query = "" - faceted_list.append(self.faceted_key(key, value)) - if len(query) > 0: - operation, query = query.split(" ", 1) - if operation == "or": - union = 1 - - else: - break - if union == 1: - get_data = self.connection.sunion(*faceted_list) - - return self.show_data(get_data) - else: - get_data = self.connection.sinter(*faceted_list) - - return self.show_data(get_data) - - def show_data(self, get_data): - """It returns the data in readable format.""" - property_list = [] - for string in get_data: - string1 = string.decode('utf-8') - property_list.append(string1) -# print("list ",property_list) - return property_list - - -class QueryFacades: - """ - It is used for call the above defined functions based on given query. - Using test as a bool which identify that function is using for tests. - """ - - def __init__(self, api_doc, url, test): - - self.endpoint_query = EndpointQuery() - self.api_doc = api_doc - self.url = url - self.properties = PropertiesQuery() - self.compare = CompareProperties() - self.test = test - - def initialize(self): - """ - Initialize is used to initialize the graph for given url. - """ - print("just initialize") - - self.graph = InitialGraph() - self.graph.main(self.url, self.api_doc) - - def user_query(self, query): - """ - It calls function based on queries type. - """ - query = query.replace("show ", "") - if query == "endpoints": - data = self.endpoint_query.get_allEndpoints(query) - return data - elif query == "classEndpoints": - data = self.endpoint_query.get_classEndpoints(query) - return data - elif query == "collectionEndpoints": - data = self.endpoint_query.get_collectionEndpoints(query) - return data - elif "members" in query: - self.members = CollectionmembersQuery(self.api_doc, - self.url, - self.graph) - if self.test: - data = self.members.data_from_server( - query.replace(" members", "")) - return data - else: - data = self.members.get_members(query) - return data - elif "objects" in query: - data = self.properties.get_members_properties(query) - return data - elif "object" in query: - data = self.properties.get_object_property(query) - return data - elif "Collection" in query: - data = self.properties.get_collection_properties(query) - return data - elif "class" in query and "property_value" in query: - self.class_property = ClassPropertiesValue(self.api_doc, - self.url, - self.graph) - data = self.class_property.get_property_value(query) - return data - elif "class" in query: - data = self.properties.get_classes_properties(query) - return data - else: - data = self.compare.object_property_comparison_list(query) - return data - - -def query(apidoc, url): - """ - It uses only for query purpose. - Querying still user wants or still user enters the exit. - :param apidoc: Apidocumentation for the given url. - :param url: url given by user. - """ - api_doc = doc_maker.create_doc(apidoc) - facades = QueryFacades(api_doc, url, False) - facades.initialize() - global check_list - check_list = [] - while True: - print("press exit to quit") - query = input(">>>") - if query == "exit": - break - elif query == "help": - help() - else: - print(facades.user_query(query)) - - -def main(): - """ - Take URL as an input and make graph using initilize function. - :return: call query function for more query. - """ - url = input("url>>>") - handle_data = HandleData() - apidoc = handle_data.load_data(url + "/vocab") - while True: - if apidoc == "error": - print("enter right url") - url = input("url>>>") - apidoc = handle_data.load_data(url + "/vocab") - else: - break - return query(apidoc, url) - - -def help(): - """It prints that how user can query.""" - print("querying format") - print("for endpoint:- show endpoint") - print("for class_endpoint:- show classEndpoint") - print("for collection_endpoint:- show collectionEndpoint") - print("for members of collection_endpoint:-", - "show members") - print("for properties of any member:-", - "show object properties ") - print("for properties of objects:-show objects properties") - print("for collection properties:-", - "show properties") - print("for classes properties:- show class properties") - print("for compare properties:-show and/or ") - - -if __name__ == "__main__": - main() -# query = input() -# query = query.replace("show ","") -# operation_and_property_query(query) -# endpoints_query(query) -# class_endpoint_query("http://35.224.198.158:8080/api","classEndpoint") diff --git a/hydra-redis/redis_proxy.py b/hydra-redis/redis_proxy.py deleted file mode 100644 index c345147..0000000 --- a/hydra-redis/redis_proxy.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -This is use to provide the connection to Redis memory. -""" - -import redis - -class RedisProxy: - """ - RedisProxy is used for make a connection to the Redis. - """ - - def __init__(self): - self.connection = redis.StrictRedis(host='localhost', port=6379, db=0) - - def get_connection(self): - return self.connection diff --git a/hydra-redis/redis_setup.sh b/hydra-redis/redis_setup.sh deleted file mode 100755 index cc63c21..0000000 --- a/hydra-redis/redis_setup.sh +++ /dev/null @@ -1,24 +0,0 @@ - -#It will remove all the previous docker engine -sudo apt-get remove docker docker-engine docker.io -sudo apt-get update -# Now install docker repo -sudo apt-get install apt-transport-https ca-certificates curl software-properties-common - 2023 curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - -sudo apt-key fingerprint 0EBFCD88 -sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) \ - stable" - -# install docker-ce -sudo apt-get update -sudo apt-get install docker-ce - - -# after getting the docker-ce -git clone https://github.com/swilly22/redis-graph.git -cd redis-graph -sudo make docker -# command for run the server -sudo docker run -p 6379:6379 redislabs/redisgraph diff --git a/hydra-redis/test/querying_test.py b/hydra-redis/test/querying_test.py deleted file mode 100644 index 263b29f..0000000 --- a/hydra-redis/test/querying_test.py +++ /dev/null @@ -1,108 +0,0 @@ -import unittest -import urllib.request -import json -from hydrus.hydraspec import doc_maker -from os import sys, path -#import sys -#sys.path[0] = "/home/sandeep/gsoc_work/python-hydra-agent/hydra-redis" -#import querying_mechanism -#print(sys.path) -#import test -#import hydra_redis -PARENT_DIR = path.dirname(path.dirname(path.abspath(__file__))) -sys.path.append(PARENT_DIR) -import querying_mechanism - - -class TestQueryingMechanism(unittest.TestCase): - - def setUp(self): - print("testing start:") - - def test_1_classendpoint(self): - """Test for class endpoint""" - check_data = [['p.id', 'p.operations', 'p.properties', 'p.type'], - ['vocab:EntryPoint/Location', - "['POST'", "'PUT'", "'GET']", - "['Location']", 'Location']] - query = "show classEndpoints" - data = query_facades.user_query(query) - self.assertEqual(data,check_data) - - def test_2_collectionendpoint(self): - """Test for collection endpoint""" - check_data = ["ControllerLogCollection", - "DroneLogCollection", - "AnomalyCollection", - "DroneCollection", - "CommandCollection", - "HttpApiLogCollection", - "DatastreamCollection", - "MessageCollection"] - query = "show collectionEndpoints" - data = query_facades.user_query(query) - for check in check_data: - if check not in str(data): - self.assertTrue(False) - self.assertTrue(True) - - def test_3_CommandCollectionmember(self): - """ - Test for all Commands in CommandCollection. - Data is already stored in check_data from the static data url. - Check_data is used for compare the data retrieve by querying process. - """ - check_data = ['[]'] - query = "show CommandCollection members" - data = query_facades.user_query(query) - self.assertEqual(data[1],check_data) - - def test_4_ControllerLogCollectionmember(self): - """ - Test for all controller logs for ControllerLogCollection. - Whole object of ControllerLogCollection is stored in check data. - Check_data is used for compare the data retrieve by querying process. - """ - check_data = [{'@id': '/api/ControllerLogCollection/65', - '@type': 'ControllerLog'}, - {'@id': '/api/ControllerLogCollection/183', - '@type': 'ControllerLog'}, - {'@id': '/api/ControllerLogCollection/374', - '@type': 'ControllerLog'}] - query = "show ControllerLogCollection members" - data = query_facades.user_query(query) - # Make data searchable and comaprable. - data1 = str(data[1]).replace('"', '') - # data retrive from the memory can be distributed: - # like type can be at first position and id can be at second. - # So, check_data split in 3 parts. - # And check all parts are in data retrieve. - if str( - check_data[0]) in data1 and str( - check_data[1]) in data1 and str( - check_data[2]) in data1: - self.assertTrue(True) - else: - self.assertTrue(False) - - - def test_5_DatastreamCollectionmember(self): - """Test for all datastream with Drone ID 2""" - check_data = ['/api/DatastreamCollection/19'] - query = "show DatastreamCollection members" - data = query_facades.user_query(query) - # Here are find the datastream only for those which have DroneID 2. - query = "show DroneID 2 and type Datastream" - data = query_facades.user_query(query) - self.assertEqual(data,check_data) - - -if __name__ == "__main__": - url = "https://storage.googleapis.com/api3/api" - vocab_url = url + "/" + "vocab" - response = urllib.request.urlopen(vocab_url) - apidoc = json.loads(response.read().decode('utf-8')) - api_doc = doc_maker.create_doc(apidoc) - query_facades = querying_mechanism.QueryFacades(api_doc, url, True) - query_facades.initialize() - unittest.main() diff --git a/hydra-redis/test/redis_test.py b/hydra-redis/test/redis_test.py deleted file mode 100644 index 4a9ae38..0000000 --- a/hydra-redis/test/redis_test.py +++ /dev/null @@ -1,114 +0,0 @@ -import unittest -import redis - - -class Tests: - def entry_point(self): - """Test for testing the data stored in entrypoint endpoint""" - - print("entrypoint db=0") - r = redis.StrictRedis(host='localhost', port=6379, db=0) - reply = r.execute_command('GRAPH.QUERY', - 'apidoc', "MATCH (p:id) RETURN p") - property_list = [] - flag = 0 - for objects in reply: - for obj in objects: - if flag == 0: - string = obj.decode('utf-8') - map_string = map(str.strip, string.split(',')) - property_list = list(map_string) - check = property_list.pop() - property_list.append(check.replace("\x00", "")) - print(property_list) - flag += 1 - break - if ("p.id" in property_list and - "p.url" in property_list and - "p.supportedOperation" in property_list): - return True - else: - return False - - def collection_endpoints(self): - """Test for testing the data stored in collection endpoints""" - - print("collection endpoints db=0") - r = redis.StrictRedis(host='localhost', port=6379, db=0) - reply = r.execute_command('GRAPH.QUERY', - 'apidoc', "MATCH (p:collection) RETURN p") - property_list = [] - flag = 0 - for objects in reply: - for obj in objects: - if flag == 0: - string = obj.decode('utf-8') - map_string = map(str.strip, string.split(',')) - property_list = list(map_string) - check = property_list.pop() - property_list.append(check.replace("\x00", "")) - print(property_list) - flag += 1 - break - if ("p.id" in property_list and - "p.operations" in property_list and - "p.members" in property_list): - return True - else: - return False - - def class_endpoints(self): - """Test for testing the data stored in classes endpoints""" - - print("class endpoints db=0") - r = redis.StrictRedis(host='localhost', port=6379, db=0) - reply = r.execute_command('GRAPH.QUERY', - 'apidoc', "MATCH (p:classes) RETURN p") - property_list = [] - flag = 0 - for objects in reply: - for obj in objects: - if flag == 0: - string = obj.decode('utf-8') - map_string = map(str.strip, string.split(',')) - property_list = list(map_string) - check = property_list.pop() - property_list.append(check.replace("\x00", "")) - print(property_list) - flag += 1 - break - if ("p.id" in property_list and - "p.operations" in property_list and - "p.properties" in property_list and - "p.type" in property_list): - return True - else: - return False - - -class TestRedisStructure(unittest.TestCase): - test_redis = Tests() - - @classmethod - def setUpClass(cls): - cls.test_database=redis.StrictRedis(host='localhost', port=6379, db=5) - cls.test_database.set("foo","bar") - cls.test_database.set("hydra","redis") - print("setUpClass db=5 keys:",cls.test_database.keys()) - - def test_entrypoint(self): - self.assertTrue(self.test_redis.entry_point()) - - def test_collectionEndpoints(self): - self.assertTrue(self.test_redis.collection_endpoints()) - - def test_classEndpoints(self): - self.assertTrue(self.test_redis.class_endpoints()) - - @classmethod - def tearDownClass(cls): - cls.test_database.flushdb() - print("tearDownClass db=5 keys:",cls.test_database.get("foo")) - -if __name__ == '__main__': - unittest.main() From 24e2b1aa01c5a6ea4498abea7b5cf55852c6c03f Mon Sep 17 00:00:00 2001 From: sandeepsajan0 Date: Mon, 9 Jul 2018 09:56:44 +0530 Subject: [PATCH 15/15] add setup and teardown method --- hydra_redis/test/querying_test.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/hydra_redis/test/querying_test.py b/hydra_redis/test/querying_test.py index 92027f1..5ce1dcb 100644 --- a/hydra_redis/test/querying_test.py +++ b/hydra_redis/test/querying_test.py @@ -1,6 +1,7 @@ import unittest import urllib.request import json +import redis from hydrus.hydraspec import doc_maker from os import sys, path from hydra_redis import querying_mechanism @@ -8,7 +9,14 @@ class TestQueryingMechanism(unittest.TestCase): def setUp(self): - print("testing start:") + url = "https://storage.googleapis.com/api3/api" + vocab_url = url + "/" + "vocab" + response = urllib.request.urlopen(vocab_url) + apidoc = json.loads(response.read().decode('utf-8')) + api_doc = doc_maker.create_doc(apidoc) + self.query_facades = querying_mechanism.QueryFacades(api_doc, url, True) + self.query_facades.initialize() + self.test_database = redis.StrictRedis(host='localhost', port=6379, db=5) def test_1_classendpoint(self): """Test for class endpoint""" @@ -17,7 +25,7 @@ def test_1_classendpoint(self): "['POST'", "'PUT'", "'GET']", "['Location']", 'Location']] query = "show classEndpoints" - data = query_facades.user_query(query) + data = self.query_facades.user_query(query) self.assertEqual(data,check_data) def test_2_collectionendpoint(self): @@ -31,7 +39,7 @@ def test_2_collectionendpoint(self): "DatastreamCollection", "MessageCollection"] query = "show collectionEndpoints" - data = query_facades.user_query(query) + data = self.query_facades.user_query(query) for check in check_data: if check not in str(data): self.assertTrue(False) @@ -45,7 +53,7 @@ def test_3_CommandCollectionmember(self): """ check_data = ['[]'] query = "show CommandCollection members" - data = query_facades.user_query(query) + data = self.query_facades.user_query(query) self.assertEqual(data[1],check_data) def test_4_ControllerLogCollectionmember(self): @@ -61,7 +69,7 @@ def test_4_ControllerLogCollectionmember(self): {'@id': '/api/ControllerLogCollection/374', '@type': 'ControllerLog'}] query = "show ControllerLogCollection members" - data = query_facades.user_query(query) + data = self.query_facades.user_query(query) # Make data searchable and comaprable. data1 = str(data[1]).replace('"', '') # data retrive from the memory can be distributed: @@ -81,19 +89,14 @@ def test_5_DatastreamCollectionmember(self): """Test for all datastream with Drone ID 2""" check_data = ['/api/DatastreamCollection/19'] query = "show DatastreamCollection members" - data = query_facades.user_query(query) + data = self.query_facades.user_query(query) # Here are find the datastream only for those which have DroneID 2. query = "show DroneID 2 and type Datastream" - data = query_facades.user_query(query) + data = self.query_facades.user_query(query) self.assertEqual(data,check_data) + def tearDown(self): + self.test_database.flushdb() if __name__ == "__main__": - url = "https://storage.googleapis.com/api3/api" - vocab_url = url + "/" + "vocab" - response = urllib.request.urlopen(vocab_url) - apidoc = json.loads(response.read().decode('utf-8')) - api_doc = doc_maker.create_doc(apidoc) - query_facades = querying_mechanism.QueryFacades(api_doc, url, True) - query_facades.initialize() unittest.main()