Merge Main to Develop

README.md

@@ -1,4 +1,42 @@
-# commit
-## commit
+<p align="center">
+  <a href="https://github.com/The-Commit-Company/commit">
+  <img src="dashboard/src/assets/commit-logo.png" alt="Commit logo" height="100" />
+     </a>
 
-The Commit Company
+   <h3 align="center">commit</h3>
+  <p align="center">Developer tooling for the Frappeverse 🪐
+     <br />
+    <br />
+    <a href="https://frappecloud.com/marketplace/apps/commit"><strong>Install on Frappe Cloud»</strong></a>
+    <br />
+    <br />
+    <a href="https://commit.frappe.cloud/"><strong>Learn More »</strong></a>
+    <br />
+    <br />
+    <a href="https://github.com/The-Commit-Company/commit/issues">Issues</a>
+    .
+    <a href="https://github.com/sponsors/The-Commit-Company?frequency=one-time">Sponsor Us!</a>
+  </p>
+</p>
+<p align="center">
+  <a href="https://github.com/The-Commit-Company/commit/blob/main/LICENSE">
+    <img alt="license" src="https://img.shields.io/badge/license-AGPLv3-blue">
+  </a>
+     <a href="https://github.com/The-Commit-Company/commit/stargazers"><img src="https://img.shields.io/github/stars/The-Commit-Company/commit" alt="Github Stars"></a>
+     <a href="https://github.com/The-Commit-Company/commit/pulse"><img src="https://img.shields.io/github/commit-activity/m/The-Commit-Company/commit" alt="Commits-per-month"></a>
+</p>
+
+# [commit](https://commit.frappe.cloud/)
+
+Born out of a need to improve developer tooling for Frappe, "Commit" allows you to visualize your app's database schema and view all it's APIs - improving developer productivity and security of your critical applications.
+
+## Basic Installation
+
+The below guide assumes that you already have a working Frappe and Bench installation. If you do not have it, then please head over to [Official Installation Guide](https://frappeframework.com/docs/user/en/installation).
+
+Go ahead and create a fresh new bench
+
+- `bench init commit`
+- `bench get-app https://github.com/The-commit-company/commit`
+- `bench new-site <preferred-site-url>`
+- `bench --site <site-url> install-app commit`

commit/api/api_explorer.py

@@ -13,12 +13,24 @@ def get_apis_for_project(project_branch: str):
 
     apis = json.loads(branch_doc.whitelisted_apis).get("apis", []) if branch_doc.whitelisted_apis else []
     documentation = json.loads(branch_doc.documentation).get("apis", []) if branch_doc.documentation else []
-    print('documentation', len(documentation))
     for api in apis:
         # find the documentation for the api whose function_name equals to name and path same as path
         for doc in documentation:
+            if isinstance(doc, str):
+                try:
+                    doc = json.loads(doc)
+                except json.JSONDecodeError:
+                    frappe.log_error(f"Invalid JSON format in documentation entry: {doc}", "Commit Docs Error")
+                    continue
+
             if doc.get("function_name") == api.get("name") and doc.get("path") == api.get("api_path"):
                 api["documentation"] = doc.get("documentation")
+                api["last_updated"] = doc.get("last_updated")
+                api["is_published"] = doc.get("is_published", 0)
+                api["published_on"] = doc.get("published_on", None)
+                api["published_by"] = doc.get("published_by", None)
+                api['publish_id'] = doc.get('publish_id', None)
+                api['published_route'] = doc.get('published_route', None)
                 break
 
     app_name, organization, app_logo = frappe.db.get_value("Commit Project", branch_doc.project, ["app_name", "org", "image"])
@@ -33,7 +45,8 @@ def get_apis_for_project(project_branch: str):
         "org_logo": org_logo,
         "branch_name": branch_doc.branch_name,
         "project_branch": branch_doc.name,
-        "last_updated": branch_doc.last_fetched
+        "last_updated": branch_doc.last_fetched,
+        'path_to_folder':branch_doc.path_to_folder
     }
 
 

commit/api/bruno.py

@@ -1,6 +1,6 @@
 import frappe
 
-@frappe.whitelist()
+@frappe.whitelist(allow_guest=True)
 def generate_bruno_file(data, return_type='download'):
     request_data = frappe.parse_json(data)
     """

commit/api/code_analysis.py

@@ -9,20 +9,23 @@ def get_name_of_app(organization, repo):
     '''
     Get name of app from repo
     '''
-    type = None
+    file_type = None
     app_name = None
     root_files = get_all_files_in_repo(access_token, organization, repo)
+    if type(root_files) == dict and root_files.get("message", "") == "Not Found":
+        return frappe.throw(f'Repository {repo} not found in organization {organization}')
+
     for file in root_files:
         if file["name"] == "pyproject.toml":
-            type = "pyproject.toml"
+            file_type = "pyproject.toml"
             break
         elif file["name"] == "setup.py":
-            type = "setup.py"
+            file_type = "setup.py"
             break
 
-    if type == "pyproject.toml":
+    if file_type == "pyproject.toml":
         app_name = get_app_name_from_pyproject_toml(organization, repo)
-    elif type == "setup.py":
+    elif file_type == "setup.py":
         app_name = get_app_name_from_setup_py(organization, repo)   
 
     return app_name

commit/api/generate_documentation.py

@@ -2,6 +2,7 @@
 import re
 from commit.commit.doctype.open_ai_settings.open_ai_settings import open_ai_call
 import frappe
+from commit.api.api_explorer import get_file_content_from_path
 
 
 def generate_docs_for_apis(api_definitions):
@@ -12,7 +13,6 @@ def generate_docs_for_apis(api_definitions):
 
     for chunk in chunks:
         chunk_docs = generate_docs_for_chunk(chunk)
-        # print("Chunk Docs:", chunk_docs)
         if chunk_docs:
             all_docs.extend(chunk_docs)
         else:
@@ -61,24 +61,22 @@ def generate_docs_for_chunk(api_chunk):
             "content": (
                 "You are an expert documentation generator. Create detailed and comprehensive documentation "
                 "for the code provided below in Markdown format. Each function should have the following sections:\n\n"
-                "- **(api[Function Name])**\n"
-                "- **Description**: Detailed description of what the function does and what it is used for \n"
-                "- **Parameters**: List of parameters with their types, descriptions, and indicate which are mandatory or optional\n"
-                "- **Return Type**: Type and description of the return value\n"
-                "- **Examples**: Code examples demonstrating how to use the function (enclosed in triple backticks ``````).\n\n"
+                "- # [Function Name] (as heading 1)\n"
+                "- ## Description: Detailed description of what the function does and what it is used for \n"
+                "- ## Parameters: List of parameters with their types, descriptions, and indicate which are mandatory or optional\n"
+                "- ## Return Type: Type and description of the return value\n"
+                "- ## Examples\n Provide code examples demonstrating how to use the function, enclosed in triple backticks (``````).\n\n"
                 "The response should be a valid JSON list of objects formatted as follows: "
-                "{function_name: <function_name>, path: <path>, documentation: <documentation in Markdown>}.\n"
-                "Ensure the response is in JSON format only, enclosed in triple backticks, and does not include `---`."
+                "{function_name: <function_name>, path: <path>, last_updated:<last_updated>, documentation: <documentation in Markdown>}.\n"
+                "Ensure the response is in valid JSON format only, enclosed in triple backticks, and does not include `---`."
             )
         }
     ]
-
+    last_updated = frappe.utils.now()
     for api in api_chunk:
-        user_message = f"function name: {api['function_name']}, path: {api['path']}, code:\n{api['code']}"
+        user_message = f"function name: {api['function_name']}, path: {api['path']}, last_updated:{last_updated} ,code:\n{api['code']}"
         messages.append({"role": "user", "content": user_message})
 
-    # print("Raw Response:\n", response_text)  # Log raw response for debugging
-
     response_text = open_ai_call(messages)
 
     cleaned_response = clean_response(response_text)
@@ -89,14 +87,155 @@ def generate_docs_for_chunk(api_chunk):
             return cleaned_response
         # If cleaned_response is a string, attempt to decode it as JSON
         elif isinstance(cleaned_response, str):
-            return json.loads(cleaned_response)
+            return json.loads(cleaned_response, strict=False)
         else:
             # Handle other unexpected types if necessary
             print("Unexpected type of cleaned_response:", type(cleaned_response))
             return []
     except json.JSONDecodeError as e:
         print("JSON Decode Error:", e)
         print("Cleaned Response:\n", cleaned_response)
+        try:
+            # Attempt to fix common issues like single quotes or trailing commas
+            cleaned_response = cleaned_response.replace("'", '"')
+            return json.loads(cleaned_response, strict=False)
+        except json.JSONDecodeError as e:
+            print("Second JSON Decode Error:", e)
+            return []
+    # return cleaned_response
+
+def generate_documentation_for_api_snippet(api_path:str,code_snippet:str):
+    messages = [
+        {
+            "role": "system",
+            "content": (
+                "You are an expert documentation generator. Create detailed and comprehensive documentation "
+                "for the code provided below in Markdown format. Each function should have the following sections:\n\n"
+                "- # [Function Name] (as heading 1)\n"
+                "- ## Description\n Provide a detailed description of what the function does and what it is used for.\n"
+                "- ## Parameters\n List of parameters with their types, descriptions, and indicate which are mandatory or optional.\n"
+                "- ## Return Type\n Specify the type and description of the return value.\n"
+                "- ## Examples\n Provide code examples demonstrating how to use the function, enclosed in triple backticks (``````).\n\n"
+                "The response should be a valid JSON formatted as follows: "
+                "{function_name: <function_name>, path: <path>, last_updated:<last_updated>, documentation: <documentation in Markdown>}.\n"
+                "Ensure the response is in valid JSON format only, and does not include `---`."
+            )
+        }
+    ]
+
+    user_message = f"api path: {api_path}, last_updated:{frappe.utils.now()}, code:\n{code_snippet}"
+    if not code_snippet:
         return []
+    messages.append({"role": "user", "content": user_message})
+
+    response_text = open_ai_call(messages)
+
+    cleaned_response = response_text
+
+    try:
+        # Check if cleaned_response is already a list (or the expected type)
+        if isinstance(cleaned_response, list):
+            return cleaned_response
+        # If cleaned_response is a string, attempt to decode it as JSON
+        elif isinstance(cleaned_response, str):
+            return json.loads(cleaned_response, strict=False)
+        else:
+            # Handle other unexpected types if necessary
+            print("Unexpected type of cleaned_response:", type(cleaned_response))
+            return []
+    except json.JSONDecodeError as e:
+        print("JSON Decode Error:", e)
+        print("Cleaned Response:\n", cleaned_response)
+        try:
+            # Attempt to fix common issues like single quotes or trailing commas
+            cleaned_response = cleaned_response.replace("'", '"')
+            return json.loads(cleaned_response, strict=False)
+        except json.JSONDecodeError as e:
+            print("Second JSON Decode Error:", e)
+            return []
 
-    # return cleaned_response
+@frappe.whitelist()
+def get_documentation_for_api(project_branch: str, file_path: str,block_start: int, block_end: int,endpoint:str,viewer_type:str = 'app'):
+    code_snippet = get_file_content_from_path(project_branch, file_path,block_start, block_end,viewer_type)
+    api_path = endpoint
+    return generate_documentation_for_api_snippet(api_path, code_snippet)
+
+@frappe.whitelist()
+def save_documentation(project_branch:str,endpoint:str,documentation:str,viewer_type:str = 'app'):
+    # Save the documentation to the project branch
+    # 1. Check for viewer_type app or project
+    # 2. If viewer_type is app, then check the document is already present in Commit Branch Documentation doctype
+    # 3. If document present then loop over documentation check if the function_name and path matches then update the documentation else create a new dict and append to the documentation
+    # 4. If document not present then create a new document and append the documentation
+    # 5. If viewer_type is project then check the document is already present in Commit Project Branch doctype
+    # 6. If document present then loop over documentation check if the function_name and path matches then update the documentation else create a new dict and append to the documentation
+    # 7. If document not present then create a new document and append the documentation
+
+    if viewer_type == "app":
+        # Check if the document is already present in Commit Branch Documentation doctype
+        save_documentation_for_site_app(project_branch, endpoint, documentation)
+    else:
+        save_documentation_for_project_branch(project_branch, endpoint, documentation)
+
+def save_documentation_for_project_branch(project_branch:str,endpoint:str,documentation:str):
+
+    doc = frappe.get_doc("Commit Project Branch", project_branch)
+    docs = json.loads(doc.documentation) if doc.documentation else {}
+    apis = docs.get("apis", [])
+
+    # apis is list of dict with keys function_name, path, last_updated, documentation
+    # loop over apis and check if function_name and path matches then update the documentation else create a new dict and append to the documentation
+    found = False
+    for api in apis:
+        if api.get("function_name") == endpoint.split(".")[-1] and api.get("path") == endpoint:
+            api["documentation"] = documentation
+            api["last_updated"] = frappe.utils.now()
+            found = True
+            break
+    if not found:
+        apis.append({
+            "function_name": endpoint.split(".")[-1],
+            "path": endpoint,
+            "last_updated": frappe.utils.now(),
+            "documentation": documentation
+        })
+
+    doc.documentation = json.dumps({"apis": apis})
+    doc.save()
+
+def save_documentation_for_site_app(project_branch:str,endpoint:str,documentation:str):
+
+    if frappe.db.exists("Commit Branch Documentation",project_branch):
+        doc = frappe.get_doc("Commit Branch Documentation", project_branch)
+        docs = json.loads(doc.documentation) if doc.documentation else {}
+        apis = docs.get("apis", [])
+
+        # apis is list of dict with keys function_name, path, last_updated, documentation
+        # loop over apis and check if function_name and path matches then update the documentation else create a new dict and append to the documentation
+        found = False
+        for api in apis:
+            if api.get("function_name") == endpoint.split(".")[-1] and api.get("path") == endpoint:
+                api["documentation"] = documentation
+                api["last_updated"] = frappe.utils.now()
+                found = True
+                break
+        if not found:
+            apis.append({
+                "function_name": endpoint.split(".")[-1],
+                "path": endpoint,
+                "last_updated": frappe.utils.now(), 
+                "documentation": documentation
+            })
+        doc.documentation = json.dumps({"apis": apis})
+        doc.save()
+    else:
+        # Create a new document and append the documentation
+        doc = frappe.new_doc("Commit Branch Documentation")
+        doc.app = project_branch
+        doc.documentation = json.dumps({"apis": [{
+            "function_name": endpoint.split(".")[-1],
+            "path": endpoint,
+            "last_updated": frappe.utils.now(),
+            "documentation": documentation
+        }]})
+        doc.save()