feat(llm): knowledge graph construction by llm

hugegraph-llm/README.md

@@ -0,0 +1,22 @@
+# hugegraph-llm
+
+## Summary
+
+The hugegraph-llm is a tool for the implementation and research related to large language models. This project includes runnable demos, it can also be used as a third-party library.
+
+As we know, graph systems can help large models address challenges like timeliness and hallucination, while large models can assist graph systems with cost-related issues.
+
+With this project, we aim to reduce the cost of using graph systems, and decrease the complexity of building knowledge graphs. This project will offers more applications and integration solutions for graph systems and large language models.
+1.  Construct knowledge graph by LLM + HugeGraph
+2.  Use natural language to operate graph databases (gremlin)
+3.  Knowledge graph supplements answer context (RAG)
+
+# Examples
+
+## Examples（knowledge graph construction by llm）
+
+1. Start the HugeGraph database, you can do it via Docker. Refer to this [link](https://hub.docker.com/r/hugegraph/hugegraph) for guidance
+2. Run example like python hugegraph-llm/examples/build_kg_test.py
+
+Note: If you need a proxy to access OpenAI's API, please set your HTTP proxy in `build_kg_test.py`.
+

hugegraph-llm/examples/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

hugegraph-llm/examples/build_kg_test.py

@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import os
+from src.operators.build_kg_operator import KgBuilder
+from src.operators.llm.openai_llm import OpenAIChat
+
+if __name__ == "__main__":
+    #  If you need a proxy to access OpenAI's API, please set your HTTP proxy here
+    os.environ["http_proxy"] = ""
+    os.environ["https_proxy"] = ""
+    api_key = ""
+
+    default_llm = OpenAIChat(
+        api_key=api_key, model_name="gpt-3.5-turbo-16k", max_tokens=4000
+    )
+    text = (
+        "Meet Sarah, a 30-year-old attorney, and her roommate, James, whom she's shared a home with since 2010. James, "
+        "in his professional life, works as a journalist. Additionally, Sarah is the proud owner of the website "
+        "www.sarahsplace.com, while James manages his own webpage, though the specific URL is not mentioned here. "
+        "These two individuals, Sarah and James, have not only forged a strong personal bond as roommates but have "
+        "also carved out their distinctive digital presence through their respective webpages, showcasing their "
+        "varied interests and experiences."
+    )
+    builder = KgBuilder(default_llm)
+    # build kg with only text
+    builder.parse_text_to_data(text).disambiguate_data().commit_data_to_kg().run()
+    # build kg with text and schemas
+    nodes_schemas = [
+        {
+            "label": "Person",
+            "primary_key": "name",
+            "properties": {"age": "int", "name": "text", "occupation": "text"},
+        },
+        {
+            "label": "Webpage",
+            "primary_key": "name",
+            "properties": {"name": "text", "url": "text"},
+        },
+    ]
+    relationships_schemas = [
+        {
+            "start": "Person",
+            "end": "Person",
+            "type": "roommate",
+            "properties": {"start": "int"},
+        },
+        {"start": "Person", "end": "Webpage", "type": "owns", "properties": {}},
+    ]
+    (
+        builder.parse_text_to_data_with_schemas(
+            text, nodes_schemas, relationships_schemas
+        )
+        .disambiguate_data_with_schemas()
+        .commit_data_to_kg()
+        .run()
+    )

hugegraph-llm/src/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

hugegraph-llm/src/operators/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

hugegraph-llm/src/operators/build_kg/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

hugegraph-llm/src/operators/build_kg/commit_data_to_kg.py

@@ -0,0 +1,188 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import os
+from hugegraph.connection import PyHugeGraph
+
+
+def generate_new_relationships(nodes_schemas_data, relationships_data):
+    label_id = dict()
+    i = 1
+    old_label = []
+    for item in nodes_schemas_data:
+        label = item["label"]
+        if label in old_label:
+            continue
+        else:
+            label_id[label] = i
+            i += 1
+            old_label.append(label)
+    new_relationships_data = []
+    for relationship in relationships_data:
+        start = relationship["start"]
+        end = relationship["end"]
+        relationships_type = relationship["type"]
+        properties = relationship["properties"]
+        new_start = []
+        new_end = []
+        for key, value in label_id.items():
+            for key1, value1 in start.items():
+                if key1 == key:
+                    new_start = f"{value}" + ":" + f"{value1}"
+            for key1, value1 in end.items():
+                if key1 == key:
+                    new_end = f"{value}" + ":" + f"{value1}"
+        relationships_data = dict()
+        relationships_data["start"] = new_start
+        relationships_data["end"] = new_end
+        relationships_data["type"] = relationships_type
+        relationships_data["properties"] = properties
+        new_relationships_data.append(relationships_data)
+    return new_relationships_data
+
+
+def generate_schema_properties(data):
+    schema_properties_statements = []
+    if len(data) == 3:
+        for item in data:
+            properties = item["properties"]
+            for key, value in properties.items():
+                if value == "int":
+                    schema_properties_statements.append(
+                        f"schema.propertyKey('{key}').asInt().ifNotExist().create()"
+                    )
+                elif value == "text":
+                    schema_properties_statements.append(
+                        f"schema.propertyKey('{key}').asText().ifNotExist().create()"
+                    )
+    else:
+        for item in data:
+            properties = item["properties"]
+            for key, value in properties.items():
+                if value == "int":
+                    schema_properties_statements.append(
+                        f"schema.propertyKey('{key}').asInt().ifNotExist().create()"
+                    )
+                elif value == "text":
+                    schema_properties_statements.append(
+                        f"schema.propertyKey('{key}').asText().ifNotExist().create()"
+                    )
+    return schema_properties_statements
+
+
+def generate_schema_nodes(data):
+    schema_nodes_statements = []
+    for item in data:
+        label = item["label"]
+        primary_key = item["primary_key"]
+        properties = item["properties"]
+        schema_statement = f"schema.vertexLabel('{label}').properties("
+        schema_statement += ", ".join(f"'{prop}'" for prop in properties.keys())
+        schema_statement += f").nullableKeys("
+        schema_statement += ", ".join(
+            f"'{prop}'" for prop in properties.keys() if prop != primary_key
+        )
+        schema_statement += (
+            f").usePrimaryKeyId().primaryKeys('{primary_key}').ifNotExist().create()"
+        )
+        schema_nodes_statements.append(schema_statement)
+    return schema_nodes_statements
+
+
+def generate_schema_relationships(data):
+    schema_relationships_statements = []
+    for item in data:
+        start = item["start"]
+        end = item["end"]
+        schema_relationships_type = item["type"]
+        properties = item["properties"]
+        schema_statement = f"schema.edgeLabel('{schema_relationships_type}').sourceLabel('{start}').targetLabel('{end}').properties("
+        schema_statement += ", ".join(f"'{prop}'" for prop in properties.keys())
+        schema_statement += f").nullableKeys("
+        schema_statement += ", ".join(f"'{prop}'" for prop in properties.keys())
+        schema_statement += f").ifNotExist().create()"
+        schema_relationships_statements.append(schema_statement)
+    return schema_relationships_statements
+
+
+def generate_nodes(data):
+    nodes = []
+    for item in data:
+        label = item["label"]
+        properties = item["properties"]
+        nodes.append(f"g.addVertex('{label}', {properties})")
+    return nodes
+
+
+def generate_relationships(data):
+    relationships = []
+    for item in data:
+        start = item["start"]
+        end = item["end"]
+        types = item["type"]
+        properties = item["properties"]
+        relationships.append(f"g.addEdge('{types}', '{start}', '{end}', {properties})")
+    return relationships
+
+
+class CommitDataToKg:
+    def __init__(self):
+        self.client = PyHugeGraph(
+            "127.0.0.1", "8080", user="admin", pwd="admin", graph="hugegraph"
+        )
+        self.schema = self.client.schema()
+
+    def run(self, data: dict):
+        # If you are using a http proxy, you can run the following code to unset http proxy
+        os.environ.pop("http_proxy")
+        os.environ.pop("https_proxy")
+        nodes = data["nodes"]
+        relationships = data["relationships"]
+        nodes_schemas = data["nodes_schemas"]
+        relationships_schemas = data["relationships_schemas"]
+        schema = self.schema
+        # properties schema
+        schema_nodes_properties = generate_schema_properties(nodes_schemas)
+        schema_relationships_properties = generate_schema_properties(
+            relationships_schemas
+        )
+        for schema_nodes_property in schema_nodes_properties:
+            exec(schema_nodes_property)
+
+        for schema_relationships_property in schema_relationships_properties:
+            exec(schema_relationships_property)
+
+        # nodes schema
+        schema_nodes = generate_schema_nodes(nodes_schemas)
+        for schema_node in schema_nodes:
+            exec(schema_node)
+
+        # relationships schema
+        schema_relationships = generate_schema_relationships(relationships_schemas)
+        for schema_relationship in schema_relationships:
+            exec(schema_relationship)
+
+        g = self.client.graph()
+        # nodes
+        nodes = generate_nodes(nodes)
+        for node in nodes:
+            exec(node)
+
+        # relationships
+        new_relationships = generate_new_relationships(nodes_schemas, relationships)
+        relationships_schemas = generate_relationships(new_relationships)
+        for relationship in relationships_schemas:
+            exec(relationship)