diff --git a/hydra_redis/classes_objects.py b/hydra_redis/classes_objects.py index d72340c..7ef4e18 100644 --- a/hydra_redis/classes_objects.py +++ b/hydra_redis/classes_objects.py @@ -6,29 +6,44 @@ class ClassEndpoints: """Contains all the classes endpoint and the objects""" - def __init__(self, redis_graph): + def __init__(self, redis_graph, class_endpoints): self.redis_graph = redis_graph + self.class_endpoints = class_endpoints def addNode(self, label1, alias1, properties1): - """Add node to the redis graph""" + """ + Add node to the redis graph + :param label1: label for the node. + :param alias1: alias for the node. + :param properties: properties for the node. + :return: Created Node + """ node = Node(label=label1, alias=alias1, properties=properties1) self.redis_graph.add_node(node) return node def addEdge(self, source_node, predicate, dest_node): - """Add edge between nodes in redis graph""" + """Add edge between nodes in redis graph + :param source_node: source node of the edge. + :param predicate: relationship between the source and destination node + :param dest_node: destination node of the edge. + """ edge = Edge(source_node, predicate, dest_node) self.redis_graph.add_edge(edge) def get_operation(self, api_doc, endpoint): - """Return all the supportedOperations for given endpoint""" + """Return all the supportedOperations for given endpoint + :param api_doc: Apidocumentaion for particular url. + :param endpoint: particular endpoint for getting supportedOperations. + :return: All operations for endpoint. + """ endpoint_method = [] for support_operation in api_doc.parsed_classes[ endpoint][ "class"].supportedOperation: endpoint_method.append(support_operation.method) - print("supportedOperation", endpoint_method) +# print("supportedOperation", endpoint_method) # all the operations for the object is stored in endpoint_method return str(endpoint_method) @@ -38,9 +53,13 @@ def objects_property( objects_node, new_list, no_endpoint_property, - entrypoint_node, api_doc): - """Nodes for every that property which is itself an object""" + """Nodes for every that property which is itself an object + :param objects_node: particular member or class node(parent node). + :param new_list: list of object properties. + :param no_endpoint_property: property_value for new_list properties. + :param api_doc: Apidocumentation of particular url. + """ print("for the property which is an object, should be a node") for obj in new_list: obj = obj.replace("vocab:", "") @@ -56,91 +75,145 @@ def objects_property( if support_property.title in api_doc.parsed_classes: endpoint_prop.append(support_property.title) # store all operation and property of object - node_properties["property"] = str(properties_title) - node_properties["property_value"] = str(no_endpoint_property) + node_properties["properties"] = str(properties_title) + node_properties["property_value"] = str(no_endpoint_property[obj]) node_alias = str(objects_node.alias + str(obj)).lower() # key for the node of the object node_properties["parent_id"] = str(objects_node.properties["@id"]) - object_node = self.addNode("object", node_alias, node_properties) + object_node = self.addNode( + str("object" + str(objects_node.properties["@type"])), + node_alias, + node_properties) self.addEdge(objects_node, "has" + str(obj), object_node) # set edge between the object and its parent object if endpoint_prop: self.objects_property( - object_node, endpoint_prop, entrypoint_node, api_doc) + object_node, endpoint_prop, api_doc) + + def faceted_key(self, key, value): + return ("{}".format("fs:" + key + ":" + value)) + + def faceted_indexing(self, + key, + redis_connection, + member): + for keys in member: + redis_connection.sadd(self.faceted_key(keys, member[keys]), key) + + def load_from_server( + self, + endpoint, + api_doc, + base_url, + redis_connection): + """Loads data from class endpoints like its properties values + :param endpoint: Particular endpoint for load data from server. + :param api_doc: Apidocumentation for particular url. + :param base_url: Parent url for accessing server. + :param redis_connection: connection for the Redis memory. + """ + print("check endpoint url....loading data") + member = {} + endpoint_property = [] + no_endpoint_property = {} + # new_url is url for the classes endpoint + new_url = base_url + "/" + endpoint + # retreiving data for the classes endpoint from server + response = urllib.request.urlopen(new_url) + new_file = json.loads(response.read().decode('utf-8')) + # endpoint_property store all properties which is class/object but not + # endpoint. + for support_property in api_doc.parsed_classes[ + endpoint][ + "class"].supportedProperty: + if (endpoint != support_property.title and + support_property.title not in self.class_endpoints): + if support_property.title in api_doc.parsed_classes: + endpoint_property.append(support_property.title) + + # members store the properties with its value + # members contains null if supportedProperty has no value in server + # no_endpoint_property store object value of property + # ex: values of state property. + if support_property.title in new_file: + if isinstance(new_file[support_property.title], str): + member[support_property.title] = str( + new_file[support_property.title].replace(" ", "")) + else: + no_endpoint_property[support_property.title] = new_file[ + support_property.title] + else: + member[support_property.title] = "null" + # Add the property_value in endpoint node properties. + for node in self.redis_graph.nodes.values(): + if node.alias == endpoint: + # update the properties of the node + node.properties["property_value"] = str(member) + # Use faceted index to handle with comparison in properties. + redis_connection.set((endpoint), member) + self.faceted_indexing(endpoint, redis_connection, member) + class_object_node = node + print(class_object_node) + # For creating relationship with endpoint_property elements. + if endpoint_property: + self.objects_property( + class_object_node, + endpoint_property, + no_endpoint_property, + api_doc) + # save the graph changes. + self.redis_graph.commit() def endpointclasses( self, - class_endpoints, entrypoint_node, api_doc, base_url): - """Node for every class which have an endpoint.""" + """Node for every class which have an endpoint. + :param entrypoint_node: Endtrypoint or parent node. + :param api_doc: Apidocumentation for particular url. + :param base_url: parent url for accessing server. + """ print("classes endpoint or accessing classes") endpoint_property_list = {} - # contain all endpoints which have other endpoints as a property. - for endpoint in class_endpoints: - supported_properties_list=[] - node_properties = {} + # endpoint_property_list contain all endpoints + # which have other endpoints as a property ex: State. + for endpoint in self.class_endpoints: + supported_properties_list = [] # node_properties is used for set the properties of node. - member = {} - # member contain the properties of class endpoint. - endpoint_property = [] - # it contains the property which is a non-endpoint object or class. + node_properties = {} property_list = [] - print("check endpoint url....loading data") - new_url = base_url + \ - class_endpoints[endpoint].replace("vocab:EntryPoint", "") - # url for the classes endpoint - print(new_url) - response = urllib.request.urlopen(new_url) - new_file = json.loads(response.read().decode('utf-8')) - # retreiving data for the classes endpoint from server + # store the operations for the endpoint node_properties["operations"] = self.get_operation( api_doc, endpoint) - # store the operations for the endpoint - + # supported_property_list contain all the properties of endpoint. + # property list store the properties which is endpoint as well. for support_property in api_doc.parsed_classes[ - endpoint][ - "class"].supportedProperty: + endpoint][ + "class"].supportedProperty: supported_properties_list.append(support_property.title) + # findout the properties which is also an endpoint. if endpoint != support_property.title: - if support_property.title in class_endpoints: + if support_property.title in self.class_endpoints: property_list.append(support_property.title) - elif support_property.title in api_doc.parsed_classes: - endpoint_property.append(support_property.title) - - if support_property.title in new_file: - if isinstance(new_file[support_property.title], str): - member[support_property.title] = str( - new_file[support_property.title].replace(" ", "")) - else: - no_endpoint_property=new_file[support_property.title] - else: - member[support_property.title] = "null" + endpoint_property_list[endpoint] = property_list - # member is using for storing the fetched data in node. - node_properties["@id"] = str(new_file["@id"]) - node_properties["@type"] = str(new_file["@type"]) - node_properties["property_value"] = str(member) - node_properties["properties"]= str(supported_properties_list) + # node_properties contains data to store in particular node. + node_properties["@id"] = str(self.class_endpoints[endpoint]) + node_properties["@type"] = str(endpoint) + node_properties["properties"] = str(supported_properties_list) class_object_node = self.addNode( "classes", str(endpoint), node_properties) + # set edge between the entrypoint and the class endpoint. self.addEdge(entrypoint_node, "has" + endpoint, class_object_node) - # set edge between the entrypoint and the class endpoint/object - if endpoint_property: - self.objects_property( - class_object_node, - endpoint_property, - no_endpoint_property, - entrypoint_node, - api_doc) + # for connect the nodes to endpoint which have endpoint as a property. if endpoint_property_list: for endpoint_property in endpoint_property_list: for src_node in self.redis_graph.nodes.values(): if str(endpoint_property) == src_node.alias: for endpoints in endpoint_property_list[ - endpoint_property]: + endpoint_property]: for nodes in self.redis_graph.nodes.values(): if endpoints == nodes.alias: self.addEdge( diff --git a/hydra_redis/collections_endpoint.py b/hydra_redis/collections_endpoint.py index 00da31f..0db3bc6 100644 --- a/hydra_redis/collections_endpoint.py +++ b/hydra_redis/collections_endpoint.py @@ -1,7 +1,7 @@ import urllib.request import json import re -from classes_objects import ClassEndpoints +from hydra_redis.classes_objects import ClassEndpoints class CollectionEndpoints: @@ -12,22 +12,41 @@ def __init__(self, redis_graph, class_endpoints): self.class_endpoints = class_endpoints def fetch_data(self, new_url): - """Fetching data from the server""" + """Fetching data from the server + :param new_url: url for fetching the data. + :return: loaded data. + """ response = urllib.request.urlopen(new_url) return json.loads(response.read().decode('utf-8')) + def faceted_key(self, fs, key, value): + return ("{}".format(fs + ":" + key + ":" + value)) + + def faceted_indexing(self, key, redis_connection, member): + for keys in member: + redis_connection.sadd( + self.faceted_key( + "fs", keys, member[keys]), key) + def collectionobjects( self, endpoint_collection_node, endpoint_list, new_url, - entrypoint_node, api_doc, - url): - """Creating nodes for all objects stored in collection.""" + url, + redis_connection): + """Creating nodes for all objects stored in collection. + :param endpoint_collection_node: parent/collection endpoint node. + :param endpoint_list: Members/objects of any collection. + :param new_url: parent url for members/objects + :param api_doc: Apidocumentation for particular url. + :param url: Base url given by user. + :param redis_connection: connection of Redis memory. + """ print("accesing the collection object like events or drones") if endpoint_list: - clas = ClassEndpoints(self.redis_graph) + clas = ClassEndpoints(self.redis_graph, self.class_endpoints) for endpoint in endpoint_list: node_properties = {} no_endpoint_list = [] @@ -35,31 +54,35 @@ def collectionobjects( member = {} endpoint_property_list = [] supported_property_list = [] + no_endpoint_property = {} match_obj = re.match( r'/(.*)/(.*)/(.*)?', endpoint["@id"], re.M | re.I) base_url = "/{0}/{1}/".format(match_obj.group(1), match_obj.group(2)) entrypoint_member = endpoint["@type"].lower( ) + match_obj.group(3) - print(base_url, entrypoint_member) +# print(base_url, entrypoint_member,endpoint["@type"]) member_alias = entrypoint_member # key for the object node is memeber_alias member_id = match_obj.group(3) - print("member alias and id", member_alias, member_id) - new_url1 = new_url + "/" + member_id - new_file1 = self.fetch_data(new_url1) + member_url = new_url + "/" + member_id # object data retrieving from the server + new_file = self.fetch_data(member_url) for support_operation in api_doc.parsed_classes[ - endpoint["@type"] - ]["class" - ].supportedOperation: + endpoint["@type"] + ]["class" + ].supportedOperation: endpoint_method.append(support_operation.method) - node_properties["operations"] = str(endpoint_method) # all the operations for the object is stored in method + node_properties["operations"] = str(endpoint_method) + # endpoint_property_list store all properties which is class/object and also an endpoint. + # supported_property_list store all the properties. + # no_endpoint_list store all properties which is class/object + # but not endpoint. for support_property in api_doc.parsed_classes[ - endpoint["@type"] - ]["class" - ].supportedProperty: + endpoint["@type"] + ]["class" + ].supportedProperty: supported_property_list.append(support_property.title) if support_property.title in self.class_endpoints: endpoint_property_list.append( @@ -67,29 +90,40 @@ def collectionobjects( elif support_property.title in api_doc.parsed_classes: no_endpoint_list.append(support_property.title) - if support_property.title in new_file1: - if isinstance(new_file1[support_property.title], str): + # members contain all the property with value. + # it contains null value for the property which not have value in server. + # no_endpoint_properrty store value for no_endpoint_list. + if support_property.title in new_file: + if isinstance(new_file[support_property.title], str): member[support_property.title] = str( - new_file1[support_property.title].replace(" ", "")) + new_file[ + support_property.title].replace(" ", "")) else: - no_endpoint_property = new_file1[support_property.title] + no_endpoint_property[ + support_property.title] = new_file[ + support_property.title] else: member[support_property.title] = "null" node_properties["@id"] = str(endpoint["@id"]) node_properties["@type"] = str(endpoint["@type"]) + member[endpoint["@type"]] = str(endpoint["@id"]) node_properties["property_value"] = str(member) + member["type"] = str(endpoint["@type"]) + redis_connection.set((endpoint["@id"]), (member)) + self.faceted_indexing( + endpoint["@id"], redis_connection, member) node_properties["properties"] = str(supported_property_list) - collection_object_node = clas.addNode( - "objects", str(member_alias), node_properties) # add object as a node in redis - clas.addEdge(endpoint_collection_node, "has_" + - str(endpoint["@type"]), collection_object_node) + collection_object_node = clas.addNode( + str("objects" + str(endpoint["@type"])), + str(member_alias.capitalize()), + node_properties) # set an edge between the collection and its object - print( - "property of endpoint which can be class but not endpoint", - no_endpoint_list - ) + clas.addEdge(endpoint_collection_node, + "has_" + str(endpoint["@type"]), + collection_object_node) + if endpoint_property_list: for endpoint_property in endpoint_property_list: for nodes in self.redis_graph.nodes.values(): @@ -103,11 +137,49 @@ def collectionobjects( collection_object_node, no_endpoint_list, no_endpoint_property, - entrypoint_node, api_doc) + else: print("NO MEMBERS") + def load_from_server( + self, + endpoint, + api_doc, + url, + redis_connection): + """Load data or members from collection endpoint + :param endpoint: Given endpoint for load data from server. + :param api_doc: Apidocumentation for particular url. + :param url: Base url given by user. + :param redis_connection: connection to Redis memory. + """ + print( + "check url for endpoint", + url + "/" + + endpoint) + new_url = url + "/" + endpoint + # url for every collection endpoint + new_file = self.fetch_data(new_url) + # retrieving the objects from the collection endpoint + for node in self.redis_graph.nodes.values(): + if node.alias == endpoint: + node.properties["members"] = str(new_file["members"]) + # update the properties of node by its members + endpoint_collection_node = node + + self.collectionobjects( + endpoint_collection_node, + new_file["members"], + new_url, + api_doc, + url, + redis_connection + ) + self.redis_graph.commit() +# for node in self.redis_graph.nodes.values(): +# print("\n",node.alias) + def endpointCollection( self, collection_endpoint, @@ -116,40 +188,24 @@ def endpointCollection( url): """It makes a node for every collection endpoint.""" print("accessing every collection in entrypoint") - clas = ClassEndpoints(self.redis_graph) + clas = ClassEndpoints(self.redis_graph, self.class_endpoints) for endpoint in collection_endpoint: endpoint_method = [] node_properties = {} - print( - "check url for endpoint", - url + - collection_endpoint[endpoint].replace("vocab:EntryPoint", "")) - new_url = url + \ - collection_endpoint[endpoint].replace("vocab:EntryPoint", "") - # url for every collection endpoint for support_operation in api_doc.collections[ endpoint][ "collection"].supportedOperation: endpoint_method.append(support_operation.method) node_properties["operations"] = str(endpoint_method) # all the operations for the collection endpoint is stored in - print("supportedOperations",node_properties["operations"]) +# print("supportedOperations",node_properties["operations"]) node_properties["@id"] = str(collection_endpoint[endpoint]) - new_file = self.fetch_data(new_url) - # retrieving the objects from the collection endpoint - node_properties["members"] = str(new_file["members"]) + node_properties["@type"] = str(endpoint) endpoint_collection_node = clas.addNode( "collection", endpoint, node_properties) +# print(endpoint_collection_node) clas.addEdge( entrypoint_node, "has_collection", endpoint_collection_node) # set an edge between the entrypoint and collection endpoint - self.collectionobjects( - endpoint_collection_node, - new_file["members"], - new_url, - entrypoint_node, - api_doc, - url - ) diff --git a/hydra_redis/hydra_graph.py b/hydra_redis/hydra_graph.py index b91b72e..11cd1d8 100644 --- a/hydra_redis/hydra_graph.py +++ b/hydra_redis/hydra_graph.py @@ -1,89 +1,96 @@ - import redis +import redis from redisgraph import Graph, Node import urllib.request import json from hydrus.hydraspec import doc_maker import hydrus from graphviz import Digraph -from classes_objects import ClassEndpoints -from collections_endpoint import CollectionEndpoints +from hydra_redis.classes_objects import ClassEndpoints +from hydra_redis.collections_endpoint import CollectionEndpoints +class InitialGraph: + -def final_file(url): - """ Open the given url and read and load the Json data.""" - response = urllib.request.urlopen(url) - return json.loads(response.read().decode('utf-8')) + def final_file(self,url): + """Open the given url and read and load the Json data. + :param url: given url to access the data from the server. + :return: data loaded from the server. + """ + response = urllib.request.urlopen(url) + return json.loads(response.read().decode('utf-8')) -def get_apistructure(entrypoint_node, api_doc): - """ It breaks the endpoint into two parts collection and classes""" - collection_endpoints = {} - class_endpoints = {} - collection = 0 - classes = 0 - print("split entrypoint into 2 types of endpoints collection and classes") - for support_property in api_doc.entrypoint.entrypoint.supportedProperty: - if isinstance( - support_property, - hydrus.hydraspec.doc_writer.EntryPointClass): - class_endpoints[support_property.name] = support_property.id_ - collection = 1 - if isinstance( - support_property, - hydrus.hydraspec.doc_writer.EntryPointCollection): - collection_endpoints[support_property.name] = support_property.id_ - classes = 1 + def get_apistructure(self,entrypoint_node, api_doc): + """ It breaks the endpoint into two parts collection and classes""" + self.collection_endpoints = {} + self.class_endpoints = {} + print("split entrypoint into 2 types of endpoints collection and classes") + for support_property in api_doc.entrypoint.entrypoint.supportedProperty: + if isinstance( + support_property, + hydrus.hydraspec.doc_writer.EntryPointClass): + self.class_endpoints[support_property.name] = support_property.id_ - print("class_endpoints", class_endpoints) - print("collection_endpoints", collection_endpoints) - if classes == 1: - clas = ClassEndpoints(redis_graph) - clas.endpointclasses(class_endpoints, entrypoint_node, api_doc, url) + if isinstance( + support_property, + hydrus.hydraspec.doc_writer.EntryPointCollection): + self.collection_endpoints[support_property.name] = support_property.id_ - if collection == 1: - coll = CollectionEndpoints(redis_graph, class_endpoints) - coll.endpointCollection( - collection_endpoints, - entrypoint_node, - api_doc, - url) + if len(self.class_endpoints.keys())>0: + clas = ClassEndpoints(self.redis_graph, self.class_endpoints) + clas.endpointclasses(entrypoint_node, api_doc, self.url) + if len(self.collection_endpoints.keys())>0: + coll = CollectionEndpoints(self.redis_graph, self.class_endpoints) + coll.endpointCollection( + self.collection_endpoints, + entrypoint_node, + api_doc, + self.url) -def get_endpoints(api_doc): - """Create node for entrypoint""" - print("creating entrypoint node") - entrypoint_properties = {} - entrypoint_properties["@id"] = str("vocab:Entrypoint") - entrypoint_properties["url"] = str( - api_doc.entrypoint.url) + str(api_doc.entrypoint.api) - entrypoint_properties["supportedOperation"] = "GET" - entrypoint_node = Node( - label="id", - alias="Entrypoint", - properties=entrypoint_properties) - redis_graph.add_node(entrypoint_node) - return get_apistructure(entrypoint_node, api_doc) + + def get_endpoints(self,api_doc, redis_connection): + """Create node for entrypoint""" + print("creating entrypoint node") + entrypoint_properties = {} + entrypoint_properties["@id"] = str("vocab:Entrypoint") + entrypoint_properties["url"] = str( + api_doc.entrypoint.url) + str(api_doc.entrypoint.api) + entrypoint_properties["supportedOperation"] = "GET" + entrypoint_node = Node( + label="id", + alias="Entrypoint", + properties=entrypoint_properties) + self.redis_graph.add_node(entrypoint_node) + redis_connection.set("EntryPoint", entrypoint_properties) + return self.get_apistructure(entrypoint_node, api_doc) + + + def main(self,new_url,api_doc): + redis_con = redis.Redis(host='localhost', port=6379) + self.url = new_url + self.redis_graph = Graph("apidoc", redis_con) + print("loading... of graph") + self.get_endpoints(api_doc, redis_con) + print("commiting") + self.redis_graph.commit() + # creating whole the graph in redis + print("done!!!!") + # uncomment below 2 lines for getting nodes for whole graph + # for node in redis_graph.nodes.values(): + # print("\n",node.alias) + # uncomment the below lines for show the graph stored in redis + # g = Digraph('redis_graph', filename='hydra_graph.gv') + # # using graphviz for visualization of graph stored in redis + # for edge in redis_graph.edges: + # g.edge(edge.src_node.alias, edge.dest_node.alias) + # g.view() + # #see the graph generated by graphviz if __name__ == "__main__": - redis_con = redis.Redis(host='localhost', port=6379) - redis_graph = Graph("apidoc", redis_con) url = "http://35.224.198.158:8080/api" - # you also can use https://storage.googleapis.com/api2/api as url apidoc = final_file(url + "/vocab") api_doc = doc_maker.create_doc(apidoc) - get_endpoints(api_doc) - print("commiting") - redis_graph.commit() - # creating whole the graph in redis - print("done!!!!") - # uncomment below 2 lines for getting nodes for whole graph -# for node in redis_graph.nodes: -# print("\n",node) - # uncomment the below lines for show the graph stored in redis -# g = Digraph('redis_graph', filename='hydra_graph.gv') -# # using graphviz for visualization of graph stored in redis -# for edge in redis_graph.edges: -# g.edge(edge.src_node.alias, edge.dest_node.alias) -# g.view() -# #see the graph generated by graphviz + initial_graph = InitialGraph() + initial_graph.main(url, api_doc) diff --git a/hydra_redis/querying_mechanism.py b/hydra_redis/querying_mechanism.py new file mode 100644 index 0000000..a32a116 --- /dev/null +++ b/hydra_redis/querying_mechanism.py @@ -0,0 +1,537 @@ +import redis +from hydra_redis.hydra_graph import InitialGraph +import urllib.request +import json +from hydrus.hydraspec import doc_maker +from urllib.error import URLError, HTTPError +from hydra_redis.collections_endpoint import CollectionEndpoints +from hydra_redis.classes_objects import ClassEndpoints +from hydra_redis.redis_proxy import RedisProxy + + +class HandleData: + """ + Handle data is used to play with data. + It have two functions load_data and show_data. + Work of load_data is to fetch the data from server and + Work of show_data is to show the data of Redis in readable format. + """ + + def load_data(self, url): + """ + Load the data for the given url and return it. + Also handle with the HTTPError, it prints the error + :param url: url for access data from the server. + :return: loaded data + """ + try: + response = urllib.request.urlopen(url) + except HTTPError as e: + print('Error code: ', e.code) + return ("error") + except URLError as e: + print('Reason: ', e.reason) + return ("error") + else: + return json.loads(response.read().decode('utf-8')) + + def show_data(self, get_data): + """ + Make the given data readable, because now it is in binary string form. + Count is using for avoid stuffs like query internal execution time. + :param get_data: data get from the Redis memory. + """ + count = 0 + all_property_lists = [] + for objects in get_data: + count += 1 + # Show data only for odd value of count. + # because for even value it contains stuffs like time and etc. + # ex: Redis provide data like if we query class endpoint + # output like: + # [[endpoints in byte object form],[query execution time:0.5ms]] + # So with the help of count, byte object convert to string + # and also show only useful strings not the query execution time. + if count % 2 != 0: + for obj in objects: + string = obj.decode('utf-8') + map_string = map(str.strip, string.split(',')) + property_list = list(map_string) + check = property_list.pop() + property_list.append(check.replace("\x00", "")) + if property_list[0] != "NULL": + # print(property_list) + all_property_lists.append(property_list) + return all_property_lists + + + +class EndpointQuery: + """ + EndpointQuery is used for get the endpoints from the Redis. + """ + + def __init__(self): + self.redis_connection = RedisProxy() + self.handle_data = HandleData() + self.connection = self.redis_connection.get_connection() + self._data = self.handle_data + + def get_allEndpoints(self, query): + """ + It will return both type(class and collection) of endpoints. + :param query: query gets from the user, Ex: endpoints + :returns: data get from the Redis memory. + """ + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + "MATCH (p:classes) RETURN p") + self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + "MATCH (p:collection) RETURN p") + print("classEndpoints + CollectionEndpoints") + + return self._data.show_data(get_data) + + def get_classEndpoints(self, query): + """ + It will return all class Endpoints. + :param query: query get from user, Ex: classEndpoint + :return: get data from the Redis memory. + """ + get_data = self.connection.execute_command( + 'GRAPH.QUERY', 'apidoc', "MATCH (p:classes) RETURN p") + + print("classEndpoints") + + return self._data.show_data(get_data) + + def get_collectionEndpoints(self, query): + """ + It will returns all collection Endpoints. + :param query: query get from the user, Ex: collectionEndpoint + :return: get data from the Redis memory. + """ + get_data = self.connection.execute_command( + 'GRAPH.QUERY', 'apidoc', "MATCH (p:collection) RETURN p") + + print("collectoinEndpoints") + + return self._data.show_data(get_data) + + +class CollectionmembersQuery: + """ + CollectionmembersQuery is used for get members of any collectionendpoint. + Once it get the data from the server and store it in Redis. + And after that it can query from Redis memory. + Check_list is using for track which collection endpoint data is in Redis. + """ + + def __init__(self, api_doc, url,graph): + self.redis_connection = RedisProxy() + self.handle_data = HandleData() + self.connection = self.redis_connection.get_connection() + self._data = self.handle_data + self.collection = CollectionEndpoints(graph.redis_graph, + graph.class_endpoints) + + self.api_doc = api_doc + self.url = url + + def data_from_server(self, endpoint): + """ + Load data from the server for first time. + :param endpoint: endpoint for getting data from the server. + :return: get data from the Redis memory. + """ + self.collection.load_from_server(endpoint, + self.api_doc, + self.url, + self.connection) + + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + 'MATCH(p:collection) WHERE(p.type="{}") RETURN p.members'.format( + endpoint)) + print(endpoint, " members") + return self._data.show_data(get_data) + + def get_members(self, query): + """ + Gets Data from the Redis. + :param query: query get from the user, Ex: DroneCollection members + :return: get data from the Redis memory. + """ + endpoint = query.replace(" members", "") + if endpoint in check_list: + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + """MATCH(p:collection) + WHERE(p.type='{}') + RETURN p.members""".format( + endpoint)) + print(endpoint, " members") + return self._data.show_data(get_data) + + else: + check_list.append(endpoint) + print(check_list) + return self.data_from_server(endpoint) + + +class PropertiesQuery: + """ + PropertiesQuery is used for all properties for alltypes of nodes like: + classes or collection endpoints, members,object. + """ + + def __init__(self): + self.redis_connection = RedisProxy() + self.handle_data = HandleData() + self.connection = self.redis_connection.get_connection() + self._data = self.handle_data + + def get_classes_properties(self, query): + """ + Show the given type of property of given Class endpoint. + :param query: get query from the user, Ex: classLocation properties + :return: get data from the Redis memory. + """ + query = query.replace("class", "") + endpoint, query = query.split(" ") + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + 'MATCH ( p:classes ) WHERE (p.type="{}") RETURN p.{}'.format( + endpoint, + query)) + print("class", endpoint, query) + return self._data.show_data(get_data) + + def get_collection_properties(self, query): + """ + Show the given type of property of given collection endpoint. + :param query: get query from the user, Ex: DroneCollection properties. + :return: get data from the Redis memory. + """ + endpoint, query = query.split(" ") + + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + 'MATCH ( p:collection ) WHERE (p.type="{}") RETURN p.{}'.format( + endpoint, + query)) + + print("collection", endpoint, query) + return self._data.show_data(get_data) + + def get_members_properties(self, query): + """ + Show the given type of property of given member. + :param query: gete query from the user, Ex: objectsDrone properties + :return: get data from the Redis memory. + """ + endpoint, query = query.split(" ") + get_data = self.connection.execute_command( + 'GRAPH.QUERY', + 'apidoc', + 'MATCH ( p:{} ) RETURN p.id,p.{}'.format( + endpoint, + query)) + + print("member", endpoint, query) + return self._data.show_data(get_data) + + def get_object_property(self, query): + """ + Show the given type of property of given object. + :param query: get query from the user,Ex:object 1: + key, value, query = query.split(" ", 2) + faceted_list.append(self.faceted_key(key, value)) + else: + key, value = query.split(" ") + query = "" + faceted_list.append(self.faceted_key(key, value)) + if len(query) > 0: + operation, query = query.split(" ", 1) + if operation == "or": + union = 1 + + else: + break + if union == 1: + get_data = self.connection.sunion(*faceted_list) + + return self.show_data(get_data) + else: + get_data = self.connection.sinter(*faceted_list) + + return self.show_data(get_data) + + def show_data(self, get_data): + """It returns the data in readable format.""" + property_list = [] + for string in get_data: + string1 = string.decode('utf-8') + property_list.append(string1) +# print("list ",property_list) + return property_list + + +class QueryFacades: + """ + It is used for call the above defined functions based on given query. + Using test as a bool which identify that function is using for tests. + """ + + def __init__(self, api_doc, url, test): + + self.endpoint_query = EndpointQuery() + self.api_doc = api_doc + self.url = url + self.properties = PropertiesQuery() + self.compare = CompareProperties() + self.test = test + + def initialize(self): + """ + Initialize is used to initialize the graph for given url. + """ + print("just initialize") + + self.graph = InitialGraph() + self.graph.main(self.url, self.api_doc) + + def user_query(self, query): + """ + It calls function based on queries type. + """ + query = query.replace("show ", "") + if query == "endpoints": + data = self.endpoint_query.get_allEndpoints(query) + return data + elif query == "classEndpoints": + data = self.endpoint_query.get_classEndpoints(query) + return data + elif query == "collectionEndpoints": + data = self.endpoint_query.get_collectionEndpoints(query) + return data + elif "members" in query: + self.members = CollectionmembersQuery(self.api_doc, + self.url, + self.graph) + if self.test: + data = self.members.data_from_server( + query.replace(" members", "")) + return data + else: + data = self.members.get_members(query) + return data + elif "objects" in query: + data = self.properties.get_members_properties(query) + return data + elif "object" in query: + data = self.properties.get_object_property(query) + return data + elif "Collection" in query: + data = self.properties.get_collection_properties(query) + return data + elif "class" in query and "property_value" in query: + self.class_property = ClassPropertiesValue(self.api_doc, + self.url, + self.graph) + data = self.class_property.get_property_value(query) + return data + elif "class" in query: + data = self.properties.get_classes_properties(query) + return data + else: + data = self.compare.object_property_comparison_list(query) + return data + + +def query(apidoc, url): + """ + It uses only for query purpose. + Querying still user wants or still user enters the exit. + :param apidoc: Apidocumentation for the given url. + :param url: url given by user. + """ + api_doc = doc_maker.create_doc(apidoc) + facades = QueryFacades(api_doc, url, False) + facades.initialize() + global check_list + check_list = [] + while True: + print("press exit to quit") + query = input(">>>") + if query == "exit": + break + elif query == "help": + help() + else: + print(facades.user_query(query)) + + +def main(): + """ + Take URL as an input and make graph using initilize function. + :return: call query function for more query. + """ + url = input("url>>>") + handle_data = HandleData() + apidoc = handle_data.load_data(url + "/vocab") + while True: + if apidoc == "error": + print("enter right url") + url = input("url>>>") + apidoc = handle_data.load_data(url + "/vocab") + else: + break + return query(apidoc, url) + + +def help(): + """It prints that how user can query.""" + print("querying format") + print("for endpoint:- show endpoint") + print("for class_endpoint:- show classEndpoint") + print("for collection_endpoint:- show collectionEndpoint") + print("for members of collection_endpoint:-", + "show members") + print("for properties of any member:-", + "show object properties ") + print("for properties of objects:-show objects properties") + print("for collection properties:-", + "show properties") + print("for classes properties:- show class properties") + print("for compare properties:-show and/or ") + + +if __name__ == "__main__": + main() +# query = input() +# query = query.replace("show ","") +# operation_and_property_query(query) +# endpoints_query(query) +# class_endpoint_query("http://35.224.198.158:8080/api","classEndpoint") diff --git a/hydra_redis/redis_proxy.py b/hydra_redis/redis_proxy.py new file mode 100644 index 0000000..c345147 --- /dev/null +++ b/hydra_redis/redis_proxy.py @@ -0,0 +1,16 @@ +""" +This is use to provide the connection to Redis memory. +""" + +import redis + +class RedisProxy: + """ + RedisProxy is used for make a connection to the Redis. + """ + + def __init__(self): + self.connection = redis.StrictRedis(host='localhost', port=6379, db=0) + + def get_connection(self): + return self.connection diff --git a/hydra_redis/test/querying_test.py b/hydra_redis/test/querying_test.py new file mode 100644 index 0000000..5ce1dcb --- /dev/null +++ b/hydra_redis/test/querying_test.py @@ -0,0 +1,102 @@ +import unittest +import urllib.request +import json +import redis +from hydrus.hydraspec import doc_maker +from os import sys, path +from hydra_redis import querying_mechanism + +class TestQueryingMechanism(unittest.TestCase): + + def setUp(self): + url = "https://storage.googleapis.com/api3/api" + vocab_url = url + "/" + "vocab" + response = urllib.request.urlopen(vocab_url) + apidoc = json.loads(response.read().decode('utf-8')) + api_doc = doc_maker.create_doc(apidoc) + self.query_facades = querying_mechanism.QueryFacades(api_doc, url, True) + self.query_facades.initialize() + self.test_database = redis.StrictRedis(host='localhost', port=6379, db=5) + + def test_1_classendpoint(self): + """Test for class endpoint""" + check_data = [['p.id', 'p.operations', 'p.properties', 'p.type'], + ['vocab:EntryPoint/Location', + "['POST'", "'PUT'", "'GET']", + "['Location']", 'Location']] + query = "show classEndpoints" + data = self.query_facades.user_query(query) + self.assertEqual(data,check_data) + + def test_2_collectionendpoint(self): + """Test for collection endpoint""" + check_data = ["ControllerLogCollection", + "DroneLogCollection", + "AnomalyCollection", + "DroneCollection", + "CommandCollection", + "HttpApiLogCollection", + "DatastreamCollection", + "MessageCollection"] + query = "show collectionEndpoints" + data = self.query_facades.user_query(query) + for check in check_data: + if check not in str(data): + self.assertTrue(False) + self.assertTrue(True) + + def test_3_CommandCollectionmember(self): + """ + Test for all Commands in CommandCollection. + Data is already stored in check_data from the static data url. + Check_data is used for compare the data retrieve by querying process. + """ + check_data = ['[]'] + query = "show CommandCollection members" + data = self.query_facades.user_query(query) + self.assertEqual(data[1],check_data) + + def test_4_ControllerLogCollectionmember(self): + """ + Test for all controller logs for ControllerLogCollection. + Whole object of ControllerLogCollection is stored in check data. + Check_data is used for compare the data retrieve by querying process. + """ + check_data = [{'@id': '/api/ControllerLogCollection/65', + '@type': 'ControllerLog'}, + {'@id': '/api/ControllerLogCollection/183', + '@type': 'ControllerLog'}, + {'@id': '/api/ControllerLogCollection/374', + '@type': 'ControllerLog'}] + query = "show ControllerLogCollection members" + data = self.query_facades.user_query(query) + # Make data searchable and comaprable. + data1 = str(data[1]).replace('"', '') + # data retrive from the memory can be distributed: + # like type can be at first position and id can be at second. + # So, check_data split in 3 parts. + # And check all parts are in data retrieve. + if str( + check_data[0]) in data1 and str( + check_data[1]) in data1 and str( + check_data[2]) in data1: + self.assertTrue(True) + else: + self.assertTrue(False) + + + def test_5_DatastreamCollectionmember(self): + """Test for all datastream with Drone ID 2""" + check_data = ['/api/DatastreamCollection/19'] + query = "show DatastreamCollection members" + data = self.query_facades.user_query(query) + # Here are find the datastream only for those which have DroneID 2. + query = "show DroneID 2 and type Datastream" + data = self.query_facades.user_query(query) + self.assertEqual(data,check_data) + + def tearDown(self): + self.test_database.flushdb() + +if __name__ == "__main__": + unittest.main()