diff --git a/github_tests_validator_app/bin/github_event_process.py b/github_tests_validator_app/bin/github_event_process.py index 9746d51..c8f72aa 100644 --- a/github_tests_validator_app/bin/github_event_process.py +++ b/github_tests_validator_app/bin/github_event_process.py @@ -10,8 +10,16 @@ from github_tests_validator_app.bin.student_challenge_results_validation import ( send_student_challenge_results, ) -from github_tests_validator_app.lib.connectors.google_sheet_connector import GSheet -from github_tests_validator_app.lib.users import GitHubUser +from github_tests_validator_app.config.config import ( + GDRIVE_MAIN_DIRECTORY_NAME, + GDRIVE_SUMMARY_SPREADSHEET, + GSHEET_DETAILS_SPREADSHEET, + USER_SHARE, +) +from github_tests_validator_app.lib.connectors.gddrive import GoogleDriveConnector +from github_tests_validator_app.lib.connectors.gsheet import GSheetConnector +from github_tests_validator_app.lib.models.file import GSheetDetailFile, GSheetFile, WorkSheetFile +from github_tests_validator_app.lib.models.users import GitHubUser from github_tests_validator_app.lib.utils import init_github_user_from_github_event process = { @@ -21,7 +29,56 @@ } -def validator(payload: Dict[str, Any]) -> Any: +def handle_process(payload: Dict[str, Any]) -> str: + # Get event + event = get_event(payload) + if ( + not event + or (event == "pull_request" and payload["action"] not in ["reopened", "opened"]) + or ( + event == "workflow_job" + and ( + payload["action"] not in ["completed"] + or payload["workflow_job"]["conclusion"] != "success" + ) + ) + ): + return "" + return event + + +def init_gsheet_file( + google_drive: GoogleDriveConnector, info: Dict[str, Any], parent_id: str, user_share: str +) -> GSheetFile: + + gsheet = google_drive.get_gsheet(info["name"], parent_id, user_share) + + list_worksheets = [ + WorkSheetFile(NAME=worksheet["name"], HEADERS=worksheet["headers"]) + for _, worksheet in info["worksheets"].items() + ] + return GSheetFile( + NAME=info["name"], + MIMETYPE=gsheet.get("mimeType", ""), + ID=gsheet.get("id", ""), + WORKSHEETS=list_worksheets, + ) + + +def init_gsheet_detail_file( + google_drive: GoogleDriveConnector, info: Dict[str, Any], parent_id: str, user_share: str +) -> GSheetDetailFile: + + gsheet = google_drive.get_gsheet(info["name"], parent_id, user_share) + return GSheetDetailFile( + NAME=info["name"], + MIMETYPE=gsheet.get("mimeType", ""), + ID=gsheet.get("id", ""), + HEADERS=info["headers"], + ) + + +def run(payload: Dict[str, Any]) -> Any: """ Validator function @@ -31,17 +88,25 @@ def validator(payload: Dict[str, Any]) -> Any: Returns: None: Return nothing """ - # Get event - event = get_event(payload) - if ( - not event - or (event == "pull_request" and payload["action"] not in ["reopened", "opened"]) - or (event == "workflow_job" and payload["action"] not in ["completed"]) - ): + + event = handle_process(payload) + if not event: return - # Init Google Sheet - gsheet = GSheet() + # Init Google Drive connector and folders + googe_drive = GoogleDriveConnector() + folder_school_of_data = googe_drive.get_gdrive_folder(GDRIVE_MAIN_DIRECTORY_NAME, USER_SHARE) + + # Init Google sheets + gsheet_summary_file = init_gsheet_file( + googe_drive, GDRIVE_SUMMARY_SPREADSHEET, folder_school_of_data["id"], USER_SHARE + ) + gsheet_details_file = init_gsheet_detail_file( + googe_drive, GSHEET_DETAILS_SPREADSHEET, folder_school_of_data["id"], USER_SHARE + ) + + # Init Google sheet connector and worksheets + gsheet = GSheetConnector(gsheet_summary_file, gsheet_details_file) # Init GitHubUser student_user = init_github_user_from_github_event(payload) @@ -49,7 +114,7 @@ def validator(payload: Dict[str, Any]) -> Any: # Logging return - # Add user on Google Sheet + # Send user on Google Sheet gsheet.add_new_user_on_sheet(student_user) # Check valid repo @@ -63,5 +128,7 @@ def validator(payload: Dict[str, Any]) -> Any: logging.error("[ERROR]: cannot get the student github repository.") return - logging.info(f"Begin {event} process...") + logging.info(f'Begin process: "{event}"...') + # Run the process process[event](student_github_connector, gsheet, payload) + logging.info(f'End of process: "{event}".') diff --git a/github_tests_validator_app/bin/github_repo_validation.py b/github_tests_validator_app/bin/github_repo_validation.py index b6467e0..0c70950 100644 --- a/github_tests_validator_app/bin/github_repo_validation.py +++ b/github_tests_validator_app/bin/github_repo_validation.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Union +from typing import Any, Dict, List, Union import logging @@ -8,19 +8,24 @@ GH_SOLUTION_REPO_NAME, GH_SOLUTION_TESTS_ACCESS_TOKEN, GH_TESTS_FOLDER_NAME, - commit_sha_path, default_message, ) from github_tests_validator_app.lib.connectors.github_connector import GitHubConnector -from github_tests_validator_app.lib.connectors.google_sheet_connector import GSheet -from github_tests_validator_app.lib.users import GitHubUser +from github_tests_validator_app.lib.connectors.gsheet import GSheetConnector +from github_tests_validator_app.lib.models.users import GitHubUser +commit_sha_path: Dict[str, List[str]] = { + "pull_request": ["pull_request", "head", "ref"], + "pusher": ["ref"], + "workflow_job": [], +} -def get_event(payload: Dict[str, Any]) -> Any: + +def get_event(payload: Dict[str, Any]) -> str: for event in commit_sha_path: if event in payload: return event - return None + return "" def get_student_branch(payload: Dict[str, Any], trigger: Union[str, None] = None) -> Any: @@ -60,9 +65,7 @@ def get_student_github_connector( if github_student_branch is None: return None - repo_name = payload["repository"]["name"] - student.get_access_token(repo_name) - return GitHubConnector(student, repo_name, github_student_branch) + return GitHubConnector(student, payload["repository"]["name"], github_student_branch) def compare_tests_folder(student_github: GitHubConnector, solution_repo: GitHubConnector) -> Any: @@ -85,13 +88,16 @@ def compare_tests_folder(student_github: GitHubConnector, solution_repo: GitHubC def github_repo_validation( - student_github_connector: GitHubConnector, gsheet: GSheet, payload: Dict[str, Any] + student_github_connector: GitHubConnector, gsheet: GSheetConnector, payload: Dict[str, Any] ) -> None: - solution_user = GitHubUser( - LOGIN=str(GH_SOLUTION_OWNER), ACCESS_TOKEN=GH_SOLUTION_TESTS_ACCESS_TOKEN + solution_user = GitHubUser(LOGIN=str(GH_SOLUTION_OWNER)) + solution_github_connector = GitHubConnector( + user=solution_user, + repo_name=GH_SOLUTION_REPO_NAME, + branch_name="main", + access_token=GH_SOLUTION_TESTS_ACCESS_TOKEN, ) - solution_github_connector = GitHubConnector(solution_user, GH_SOLUTION_REPO_NAME, "main") if not solution_github_connector: gsheet.add_new_repo_valid_result( solution_user, diff --git a/github_tests_validator_app/bin/student_challenge_results_validation.py b/github_tests_validator_app/bin/student_challenge_results_validation.py index 54d48b0..31af8da 100644 --- a/github_tests_validator_app/bin/student_challenge_results_validation.py +++ b/github_tests_validator_app/bin/student_challenge_results_validation.py @@ -1,44 +1,44 @@ from typing import Any, Dict, List, Tuple, Union import logging -from collections import defaultdict -from github_tests_validator_app.config.config import CHALLENGES_PATH +from github_tests_validator_app.config.config import CHALLENGE_DIR from github_tests_validator_app.lib.connectors.github_connector import GitHubConnector -from github_tests_validator_app.lib.connectors.google_sheet_connector import GSheet -from github_tests_validator_app.lib.pytest_result import PytestResult -from github_tests_validator_app.lib.users import GitHubUser +from github_tests_validator_app.lib.connectors.gsheet import GSheetConnector +from github_tests_validator_app.lib.models.pytest_result import PytestResult -def init_pytest_result_from_artifact(artifact: Dict[str, Any]) -> Union[PytestResult, None]: +def init_pytest_result_from_artifact( + artifact: Dict[str, Any], workflow_run_id: int +) -> Union[PytestResult, None]: if not artifact: return None - # Get result of test - return PytestResult( DURATION=artifact["duration"], TOTAL_TESTS_COLLECTED=artifact["summary"]["collected"], - TOTAL_PASSED=artifact["summary"]["passed"], - TOTAL_FAILED=artifact["summary"]["failed"], + TOTAL_PASSED_TEST=artifact["summary"]["passed"], + TOTAL_FAILED_TEST=artifact["summary"]["failed"], DESCRIPTION_TEST_RESULTS=artifact["tests"], + WORKFLOW_RUN_ID=workflow_run_id, ) def get_student_artifact( student_github_connector: GitHubConnector, - gsheet: GSheet, + gsheet: GSheetConnector, all_student_artifact: Dict[str, Any], payload: Dict[str, Any], ) -> Any: + workflow_run_id = payload["workflow_job"]["run_id"] artifact_info = student_github_connector.get_artifact_info_from_artifacts_with_worflow_run_id( all_student_artifact["artifacts"], workflow_run_id ) if not artifact_info: - gsheet.add_new_student_challenge_result( + gsheet.add_new_student_result_summary( user=student_github_connector.user, - result={}, + result=PytestResult(), info="[ERROR]: Cannot find the artifact of Pytest result on GitHub user repository.", ) logging.error( @@ -46,13 +46,13 @@ def get_student_artifact( ) return None - ### Read Artifact + # Read Artifact artifact_resp = student_github_connector.get_artifact(artifact_info) - artifact = student_github_connector.get_artifact_from_format_bytes_zip(artifact_resp.content) + artifact = student_github_connector.get_artifact_from_format_zip_bytes(artifact_resp.content) if not artifact: - gsheet.add_new_student_challenge_result( + gsheet.add_new_student_result_summary( user=student_github_connector.user, - result={}, + result=PytestResult(), info="[ERROR]: Cannot read the artifact of Pytest result on GitHub user repository.", ) logging.error( @@ -63,79 +63,54 @@ def get_student_artifact( return artifact -def get_challenge_information_from_path(path: str) -> Tuple[str, str, str]: - list_path_name = path.split(CHALLENGES_PATH)[1].split("/") - challenge_id = "-".join([name[0:2] for name in list_path_name if not ".py" in name]) - challenge_name = path[-1].split(".py")[0].split("test_")[1] - test_name = path[-1].split("::")[1] - return challenge_name, challenge_id, test_name +def get_test_information(path: str) -> Tuple[str, str, str, str]: + + list_path_name = path.split("::") + file_path = list_path_name[0] + script_name = list_path_name[0].split("/")[-1] + test_name = list_path_name[1] + challenge_id = "-".join( + [ + name[:2] + for name in list_path_name[0].split(CHALLENGE_DIR)[1].split("/") + if ".py" not in name + ] + ) + return challenge_id, file_path, script_name, test_name def parsing_challenge_results(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - # challenge_results = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(str))))) challenge_results = [] for test in results: - challenge_name, challenge_id, test_name = get_challenge_information_from_path( - test["nodeid"] - ) - # if challenge_id in challenges_ref and challenge_name in challenges_ref[challenge_id]['name']: - # challenge_results[challenge_id][challenge_name]["coef"] = challenges_ref[challenge_id]["coef"] - # challenge_results[challenge_id][challenge_name]["tests"][test_name]["result"] = 1 if test.pop("outcome") == "passed" else 0 - - # challenge_results[challenge_id][challenge_name]["tests"][test_name]["result"] = test.pop("outcome") - # challenge_results[challenge_id][challenge_name]["tests"][test_name]["setup"] = test["setup"] - # challenge_results[challenge_id][challenge_name]["tests"][test_name]["call"] = test["call"] - # challenge_results[challenge_id][challenge_name]["tests"][test_name]["teardown"] = test["teardown"] - info = {"setup": test["setup"], "call": test["call"], "teardown": test["teardown"]} - + challenge_id, file_path, script_name, test_name = get_test_information(test["nodeid"]) challenge_results.append( { - "id": challenge_id, - "script": challenge_name, - "test": test_name, - "info": info, - "result": test["outcome"], - "path": test["nodeid"], + "file_path": file_path, + "script_name": script_name, + "test_name": test_name, + "challenge_id": challenge_id, + "outcome": test["outcome"], + "setup": test["setup"], + "call": test["call"], + "teardown": test["teardown"], } ) - # list_id = challenges_ref.keys() - challenge_results.keys() - # for challenge_id in list_id: - # challenge_results[challenge_id][challenge_name]["coef"] = challenges_ref[challenge_id]["coef"] - # breakpoint() return challenge_results -def get_final_results_challenges(challenge_results: Any) -> float: - - final_results = 0.0 - total_test = 0 - - for challenge_id in challenge_results: - for challenge_info in challenge_results[challenge_id].values(): - total_test += challenge_info["coef"] - passed = sum(test["result"] for test in challenge_info["tests"].values()) / len( - challenge_info["tests"] - ) - passed = passed * challenge_info["coef"] - final_results += passed - - final_results /= total_test - return final_results - - def send_student_challenge_results( - student_github_connector: GitHubConnector, gsheet: GSheet, payload: Dict[str, Any] + student_github_connector: GitHubConnector, gsheet: GSheetConnector, payload: Dict[str, Any] ) -> None: - ### Get student artifact + # Get all artifacts all_student_artifact = student_github_connector.get_all_artifacts() if not all_student_artifact: message = f"[ERROR]: Cannot get all artifact on repository {student_github_connector.REPO_NAME} of user {student_github_connector.user.LOGIN}." if all_student_artifact["total_count"] == 0: message = f"[ERROR]: No artifact on repository {student_github_connector.REPO_NAME} of user {student_github_connector.user.LOGIN}." - gsheet.add_new_student_challenge_result( + gsheet.add_new_student_result_summary( user=student_github_connector.user, result={}, info=message, @@ -143,24 +118,25 @@ def send_student_challenge_results( logging.error(message) return + # Get student artifact artifact = get_student_artifact(student_github_connector, gsheet, all_student_artifact, payload) - # challenges_ref = gsheet.get_challenge_coef() if not artifact: - # Logging error return - ### Parsing artifact / challenge results - challenge_results = parsing_challenge_results(artifact["tests"]) - ### Get final results of student challenge results - # final_result = get_final_results_challenges(challenge_results) - - # Get result - pytest_result = init_pytest_result_from_artifact(artifact) - ## Send Results to Google Sheet - gsheet.add_new_student_challenge_result( + # Get summary student results + pytest_result = init_pytest_result_from_artifact(artifact, payload["workflow_job"]["run_id"]) + # Send summary student results to Google Sheet + gsheet.add_new_student_result_summary( user=student_github_connector.user, result=pytest_result, info="Result of student tests", ) - ### Delete artifact ? + # Parsing artifact / challenge results + challenge_results = parsing_challenge_results(artifact["tests"]) + # Send new detail results to Google Sheet + gsheet.add_new_student_detail_results( + user=student_github_connector.user, + results=challenge_results, + workflow_run_id=payload["workflow_job"]["run_id"], + ) diff --git a/github_tests_validator_app/config/config.py b/github_tests_validator_app/config/config.py index 36a0c59..7df80a4 100644 --- a/github_tests_validator_app/config/config.py +++ b/github_tests_validator_app/config/config.py @@ -1,49 +1,41 @@ -from typing import Dict, List, cast +from typing import Dict, cast import os -from github import GithubIntegration - -### GitHub ### -GH_APP_ID = cast(str, os.getenv("GH_APP_ID")) -# APP_KEY = cast(str, os.getenv("GH_APP_KEY")) -GH_APP_KEY_PATH = os.getenv("GH_APP_KEY") -GH_APP_KEY = "" -if GH_APP_KEY_PATH: - with open(GH_APP_KEY_PATH) as f: - GH_APP_KEY = f.read() -GH_SOLUTION_TESTS_ACCESS_TOKEN = cast(str, os.getenv("SOLUTION_TESTS_ACCESS_TOKEN")) +import yaml + +# GitHub +GH_APP_ID = cast(str, os.getenv("GH_APP_ID", None)) +GH_APP_KEY_PATH = os.getenv("GH_APP_KEY", "") +GH_SOLUTION_TESTS_ACCESS_TOKEN = cast(str, os.getenv("SOLUTION_TESTS_ACCESS_TOKEN", None)) GH_SOLUTION_OWNER = "artefactory-fr" GH_SOLUTION_REPO_NAME = "school_of_data_tests" GH_TESTS_FOLDER_NAME = "tests" GH_API = "https://api.github.com/repos" GH_ALL_ARTIFACT_ENDPOINT = "actions/artifacts" -git_integration = GithubIntegration( - GH_APP_ID, - GH_APP_KEY, -) +# Google Drive +GDRIVE_CREDENTIALS_PATH = "credentials.json" +GDRIVE_MAIN_DIRECTORY_NAME = "school_of_data_results" # Google Sheet -GSHEET_SA_JSON = cast(str, os.getenv("GSHEET_SA_JSON")) -GSHEET_SPREADSHEET_ID = "1tzn73q_QhZ2gLAmZObRsE_JmD6yD6433uZBGc-Llsdk" -GSHEET_WORKSHEET_STUDENT = "students" -GSHEET_WORKSHEET_CHECK_VALIDATION_REPO = "check_validation_repo" -GSHEET_WORKSHEET_STUDENT_CHALLENGE_RESULT = "student_challenge_results" -GSHEET_WORKSHEET_STUDENT_CHALLENGE_REF = "student_challenge_ref" - -# Others +GSHEET_SA_JSON = cast(str, os.getenv("GSHEET_SA_JSON", None)) +GDRIVE_HIERARCHY_APTH = "github_tests_validator_app/config/data/gdrive_hierarchy.yml" +with open(GDRIVE_HIERARCHY_APTH) as file: + data = yaml.safe_load(file) + +GDRIVE_SUMMARY_SPREADSHEET = data["gdrive_summary_spreadsheet"] +GSHEET_DETAILS_SPREADSHEET = data["gsheet_details_spreadsheet"] + +# Log message default_message: Dict[str, Dict[str, str]] = { "valid_repository": { "True": "Your folder `Test` is valid", "False": "Your folder `Test` has been modified and is no longer valid.", - } -} - -commit_sha_path: Dict[str, List[str]] = { - "pull_request": ["pull_request", "head", "ref"], - "pusher": ["ref"], - "workflow_job": [], + }, } -CHALLENGES_PATH = "tests/tests/" +# Common +CHALLENGE_DIR = "tests/tests/" +DATE_FORMAT = "%d/%m/%Y %H:%M:%S" +USER_SHARE = os.getenv("USER_SHARE", "") diff --git a/github_tests_validator_app/config/data/gdrive_hierarchy.yml b/github_tests_validator_app/config/data/gdrive_hierarchy.yml new file mode 100644 index 0000000..70ddedc --- /dev/null +++ b/github_tests_validator_app/config/data/gdrive_hierarchy.yml @@ -0,0 +1,45 @@ +--- +gdrive_summary_spreadsheet: + name: "Student challenge results" + worksheets: + student: + name: students + headers: + - login + - url + - id + - created_at + check_validation_repo: + name: check_validation_repo + headers: + - login + - user_id + - is_valid + - created_at + - info + student_challenge_results: + name: student_challenge_results + headers: + - login + - workflow_run_id + - created_at + - total_tests_collected + - total_passed_test + - total_failed_test + - duration + - info + +gsheet_details_spreadsheet: + name: Details + headers: + - login + - workflow_run_id + - created_at + - file_path + - script_name + - test_name + - outcome + - challenge_id + - setup + - call + - teardown diff --git a/github_tests_validator_app/lib/connectors/gddrive.py b/github_tests_validator_app/lib/connectors/gddrive.py new file mode 100644 index 0000000..76f5777 --- /dev/null +++ b/github_tests_validator_app/lib/connectors/gddrive.py @@ -0,0 +1,181 @@ +from typing import Any, Dict, List + +import json +import logging + +from github_tests_validator_app.config.config import GDRIVE_CREDENTIALS_PATH +from google.oauth2 import service_account +from googleapiclient.discovery import Resource, build + +SCOPES = ["https://www.googleapis.com/auth/drive"] + + +class GoogleDriveConnector: + def __init__(self): + + self.creds_path = GDRIVE_CREDENTIALS_PATH + self.creds_file = json.load(open(self.creds_path)) + + logging.info(f"Connecting to Google Drive API ...") + self.creds = service_account.Credentials.from_service_account_info( + self.creds_file, scopes=SCOPES + ) + self.drive_api = build("drive", "v3", credentials=self.creds) + logging.info(f"Done.") + + def get_all_folder(self) -> Any: + """Get all folders from a google drive. + ... + :return: All folders informations. + :rtype: Any + """ + results = ( + self.drive_api.files() + .list(q="mimeType = 'application/vnd.google-apps.folder'", spaces="drive") + .execute() + ) + return results.get("files", []) + + def get_all_file(self, parent_folder_ids: str = "") -> Any: + """Get all files from a folder on Google Drive. + :param parent_folder_ids: Folder ID. + ... + :return: All file informations from a folder. + :rtype: Any + """ + query = "" + if parent_folder_ids: + query = f"parents in '{parent_folder_ids}'" + response = ( + self.drive_api.files() + .list( + q=query, + spaces="drive", + fields="nextPageToken, files(id, name, mimeType)", + pageToken=None, + ) + .execute() + ) + return response.get("files", []) + + def create_folder(self, folder_name: str) -> Any: + """Create a folder in google drive. + :param folder_name: Folder title. + ... + :return: new folder informations + :rtype: Any + """ + # create drive api client + file_metadata = {"name": folder_name, "mimeType": "application/vnd.google-apps.folder"} + + # pylint: disable=maybe-no-member + folder = self.drive_api.files().create(body=file_metadata, fields="id").execute() + id = folder.get("id") + logging.info(f'Folder {folder_name} has created with ID: "{id}".') + return folder + + def share_file(self, real_file_id: str, user_email: str) -> List[Any]: + """Share the file with new user email. + :param real_file_id: File ID. + :param user_email: email that we want to share with the file. + ... + :return: informations of the folder + :rtype: List[Any] + """ + ids = [] + file_id = real_file_id + + def callback(request_id, response, exception): + if exception: + logging.error(f"Request_Id: {request_id}") + logging.error(exception) + else: + ids.append(response.get("id")) + + batch = self.drive_api.new_batch_http_request(callback=callback) + user_permission = {"type": "user", "role": "writer", "emailAddress": user_email} + batch.add( + self.drive_api.permissions().create(fileId=file_id, body=user_permission, fields="id") + ) + batch.execute() + + return ids + + def get_gdrive_folder(self, folder_name: str, user_share: str) -> Any: + """Get the folder information in google drive. + .. note :: + If the folder doesn't exist, it will create a new one. + :param folder_name: Folder title. + :param user_share: email that we want to share with the folder. + ... + :return: informations of the folder + :rtype: Any + """ + list_dir = self.get_all_folder() + for dir in list_dir: + if dir["name"] == folder_name: + return dir + + folder = self.create_folder(folder_name) + if "id" in folder: + self.share_file(folder["id"], user_share) + return { + "name": folder_name, + "id": folder["id"], + "kind": "drive#file", + "mimeType": "application/vnd.google-apps.folder", + } + + def get_gsheet( + self, gsheet_name: str, parent_folder_ids: str = "", user_share: str = "" + ) -> Any: + """Get the google sheet information. + .. note :: + If the google sheet doesn't exist, it will create a new one. + :param gsheet_name: Google Sheet title. + :param parent_folder_ids: A list of strings of parent folder ids (if any). + :param user_share: email that we want to share with the google sheet. + ... + :return: informations of the google sheet + :rtype: Any + """ + list_file = self.get_all_file(parent_folder_ids) + for file in list_file: + if file["name"] == gsheet_name and "spreadsheet" in file["mimeType"]: + return file + file = self.create_google_file( + self.drive_api, + gsheet_name, + "application/vnd.google-apps.spreadsheet", + [parent_folder_ids], + ) + if user_share: + self.share_file(file["id"], user_share) + return file + + def create_google_file( + self, drive_api: Resource, title: str, mimeType: str, parent_folder_ids: List[str] = [] + ) -> Any: + """Create a new file on Google drive. + .. note :: + Created file is not instantly visible in your Drive search and you need to access it by direct link. + :param title: File title + :param parent_folder_ids: A list of strings of parent folder ids (if any). + ... + :return: informations of new file + :rtype: Any + """ + logging.info(f"Creating Sheet {title}") + body: Dict[str, Any] = { + "name": title, + "mimeType": mimeType, + } + + if parent_folder_ids: + body["parents"] = parent_folder_ids + + req = drive_api.files().create(body=body) + new_sheet = req.execute() + + # Get id of fresh sheet + return new_sheet diff --git a/github_tests_validator_app/lib/connectors/github_connector.py b/github_tests_validator_app/lib/connectors/github_connector.py index 3c89330..87bf5cf 100644 --- a/github_tests_validator_app/lib/connectors/github_connector.py +++ b/github_tests_validator_app/lib/connectors/github_connector.py @@ -6,23 +6,52 @@ import zipfile import requests -from github import ContentFile, Github, Repository -from github_tests_validator_app.config.config import GH_ALL_ARTIFACT_ENDPOINT, GH_API -from github_tests_validator_app.lib.users import GitHubUser +from github import ContentFile, Github, GithubIntegration, Repository +from github_tests_validator_app.config.config import ( + GH_ALL_ARTIFACT_ENDPOINT, + GH_API, + GH_APP_ID, + GH_APP_KEY_PATH, +) +from github_tests_validator_app.lib.models.users import GitHubUser from github_tests_validator_app.lib.utils import get_hash_files class GitHubConnector: - def __init__(self, user: GitHubUser, repo_name: str, branch_name: str): + def __init__( + self, + user: GitHubUser, + repo_name: str, + branch_name: str, + access_token: Union[str, None] = None, + ): self.user = user self.REPO_NAME = repo_name self.BRANCH_NAME = branch_name + self.ACCESS_TOKEN = access_token logging.info(f"Connecting to Github with user {self.user.LOGIN} on repo: {repo_name} ...") - self.connector = Github(login_or_token=self.user.ACCESS_TOKEN, timeout=30) + if not access_token: + self.set_git_integration() + self.set_access_token(repo_name) + self.connector = Github(login_or_token=self.ACCESS_TOKEN, timeout=30) self.repo = self.connector.get_repo(f"{self.user.LOGIN}/{repo_name}") logging.info("Done.") + def set_git_integration(self) -> None: + + with open(GH_APP_KEY_PATH) as f: + GH_APP_KEY = f.read() + self.git_integration = GithubIntegration( + GH_APP_ID, + GH_APP_KEY, + ) + + def set_access_token(self, repo_name: str) -> None: + self.ACCESS_TOKEN = self.git_integration.get_access_token( + self.git_integration.get_installation(self.user.LOGIN, repo_name).id + ).token + def get_repo(self, repo_name: str) -> Repository.Repository: self.REPO_NAME = repo_name logging.info(f"Connecting to new repo: {repo_name} with user: {self.user.LOGIN} ...") @@ -64,7 +93,7 @@ def get_artifact_info_from_artifacts_with_worflow_run_id( return artifact return None - def get_artifact_from_format_bytes_zip(self, artifact_content: bytes) -> Any: + def get_artifact_from_format_zip_bytes(self, artifact_content: bytes) -> Any: z = zipfile.ZipFile(io.BytesIO(artifact_content)) f = z.read(z.namelist()[0]) decode = f.decode("utf-8") @@ -88,12 +117,12 @@ def get_artifact(self, artifact_info: Dict[str, Any]) -> Union[requests.models.R return response def _get_headers(self) -> Dict[str, str]: - if not self.user.ACCESS_TOKEN: - self.user.get_access_token(self.REPO_NAME) + if not self.ACCESS_TOKEN: + self.set_access_token(self.REPO_NAME) return { "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {self.user.ACCESS_TOKEN}", + "Authorization": f"Bearer {self.ACCESS_TOKEN}", } def _request_data( diff --git a/github_tests_validator_app/lib/connectors/google_sheet_connector.py b/github_tests_validator_app/lib/connectors/google_sheet_connector.py deleted file mode 100644 index 48d8913..0000000 --- a/github_tests_validator_app/lib/connectors/google_sheet_connector.py +++ /dev/null @@ -1,100 +0,0 @@ -from typing import Any, DefaultDict, List - -import logging -from collections import defaultdict - -import gspread -from github_tests_validator_app.config.config import ( - GSHEET_SA_JSON, - GSHEET_SPREADSHEET_ID, - GSHEET_WORKSHEET_CHECK_VALIDATION_REPO, - GSHEET_WORKSHEET_STUDENT, - GSHEET_WORKSHEET_STUDENT_CHALLENGE_REF, - GSHEET_WORKSHEET_STUDENT_CHALLENGE_RESULT, -) -from github_tests_validator_app.lib.pytest_result import PytestResult -from github_tests_validator_app.lib.users import GitHubUser - - -class GSheet: - def __init__(self): - logging.info(f"Connecting to Google Sheet API ...") - self.gs_client = gspread.service_account(filename=GSHEET_SA_JSON) - self.spreadsheet = self.gs_client.open_by_key(GSHEET_SPREADSHEET_ID) - logging.info("Done.") - - def get_new_sheet(self, sheet_id: str) -> gspread.spreadsheet.Spreadsheet: - self.spreadsheet = self.gs_client.open_by_key(sheet_id) - return self.spreadsheet - - def add_new_user_on_sheet(self, user: GitHubUser) -> None: - # Controle the workseet exist of not - worksheet = self.spreadsheet.worksheet(GSHEET_WORKSHEET_STUDENT) - - # Check is user exist - id_cell = worksheet.find(str(user.ID)) - login_cell = worksheet.find(user.LOGIN) - if id_cell and login_cell and id_cell.row == login_cell.row: - logging.info("User already exist in student worksheet.") - else: - logging.info(f"Add new user {user.LOGIN} in student worksheet ...") - headers = worksheet.row_values(1) - user_dict = user.__dict__ - new_row = [ - user_dict[header.upper()] if header.upper() in user_dict else None - for header in headers - ] - worksheet.append_row(new_row) - logging.info("Done.") - - def add_new_repo_valid_result(self, user: GitHubUser, result: bool, info: str = "") -> None: - worksheet = self.spreadsheet.worksheet(GSHEET_WORKSHEET_CHECK_VALIDATION_REPO) - headers = worksheet.row_values(1) - user_dict = user.__dict__ - new_row = list() - for header in headers: - if header == "is_valid": - new_row.append(str(result)) - elif header == "user_id": - new_row.append(user.ID) - elif header == "info": - new_row.append(info) - elif header.upper() in user_dict: - new_row.append(user_dict[header.upper()]) - else: - new_row.append("") - worksheet.append_row(new_row) - - def add_new_student_challenge_result( - self, user: GitHubUser, result: PytestResult, info: str = "" - ) -> None: - worksheet = self.spreadsheet.worksheet(GSHEET_WORKSHEET_STUDENT_CHALLENGE_RESULT) - headers = worksheet.row_values(1) - user_dict = user.__dict__ - result_dict = result.__dict__ - new_row = list() - for header in headers: - - if header.upper() in user_dict: - new_row.append(user_dict[header.upper()]) - elif header.upper() in result_dict: - new_row.append(result_dict[header.upper()]) - elif header == "info": - new_row.append(info) - else: - new_row.append("") - worksheet.append_row(new_row) - - def get_challenge_coef(self) -> DefaultDict[str, DefaultDict[str, Any]]: - - worksheet = self.spreadsheet.worksheet(GSHEET_WORKSHEET_STUDENT_CHALLENGE_REF) - dict_results = defaultdict( - lambda: defaultdict(list) - ) # type: DefaultDict[str, DefaultDict[str, Any]] - for row in worksheet.get_all_records(): - id = row.pop("id") - breakpoint() - dict_results[id]["name"].append(row.pop("challenge_name")) - dict_results[id] = defaultdict(defaultdict, {**dict_results[id], **row}) - - return dict_results diff --git a/github_tests_validator_app/lib/connectors/gsheet.py b/github_tests_validator_app/lib/connectors/gsheet.py new file mode 100644 index 0000000..01d701b --- /dev/null +++ b/github_tests_validator_app/lib/connectors/gsheet.py @@ -0,0 +1,148 @@ +from typing import Any, Dict, List + +import json +import logging + +import gspread +from github_tests_validator_app.config.config import GDRIVE_SUMMARY_SPREADSHEET, GSHEET_SA_JSON +from github_tests_validator_app.lib.models.file import GSheetDetailFile, GSheetFile +from github_tests_validator_app.lib.models.pytest_result import PytestResult +from github_tests_validator_app.lib.models.users import GitHubUser + + +class GSheetConnector: + def __init__(self, gsheet_summary_file: GSheetFile, gsheet_details_file: GSheetDetailFile): + self.gsheet_summary_file = gsheet_summary_file + self.gsheet_details_file = gsheet_details_file + + logging.info(f"Connecting to Google Sheet API ...") + self.gs_client = gspread.service_account(filename=GSHEET_SA_JSON) + logging.info("Done.") + + logging.info(f"Init spreadsheet ...") + self.summary_spreadsheet = self.init_spreadsheet(gsheet_summary_file) + self.detail_spreadsheet = self.gs_client.open_by_key(gsheet_details_file.ID) + logging.info(f"Done.") + + def add_worksheet( + self, spreadsheet: gspread.spreadsheet.Spreadsheet, title: str, headers: List[str] + ) -> gspread.worksheet.Worksheet: + + new_worksheet = spreadsheet.add_worksheet(title=title, rows=1, cols=1) + new_worksheet.insert_row(headers) + return new_worksheet + + def init_spreadsheet(self, gsheet_file: GSheetFile) -> gspread.spreadsheet.Spreadsheet: + + spreadsheet = self.gs_client.open_by_key(gsheet_file.ID) + all_worksheets = spreadsheet.worksheets() + all_worksheets_name = [worksheet.title for worksheet in all_worksheets] + + # Init all worksheets + for worksheet in gsheet_file.WORKSHEETS: + + if worksheet.NAME in all_worksheets_name: + continue + + if all_worksheets and all_worksheets[0].title == "Sheet1": + new_worksheet = all_worksheets.pop(0) + new_worksheet.update_title(worksheet.NAME) + new_worksheet.insert_row(worksheet.HEADERS) + else: + self.add_worksheet(spreadsheet, worksheet.NAME, worksheet.HEADERS) + return spreadsheet + + def add_new_user_on_sheet(self, user: GitHubUser) -> None: + # Controle the workseet exist of not + worksheet = self.summary_spreadsheet.worksheet( + GDRIVE_SUMMARY_SPREADSHEET["worksheets"]["student"]["name"] + ) + + # Check is user exist + id_cell = worksheet.find(str(user.ID)) + login_cell = worksheet.find(user.LOGIN) + if id_cell and login_cell and id_cell.row == login_cell.row: + logging.info("User already exist in student worksheet.") + else: + logging.info(f"Add new user {user.LOGIN} in student worksheet ...") + headers = worksheet.row_values(1) + user_dict = user.__dict__ + new_row = [ + user_dict[header.upper()] if header.upper() in user_dict else None + for header in headers + ] + worksheet.append_row(new_row) + logging.info("Done.") + + def dict_to_row( + self, headers: List[str], data: Dict[str, Any], to_str: bool = False, **kwargs: Any + ) -> List[str]: + result = [] + for header in headers: + value: Any = "" + if header in data: + value = data[header] + elif header in kwargs: + value = kwargs[header] + if to_str and isinstance(value, dict): + value = json.dumps(value) + result.append(value) + return result + + def add_new_repo_valid_result(self, user: GitHubUser, result: bool, info: str = "") -> None: + worksheet = self.summary_spreadsheet.worksheet( + GDRIVE_SUMMARY_SPREADSHEET["worksheets"]["check_validation_repo"]["name"] + ) + headers = worksheet.row_values(1) + user_dict = {k.lower(): v for k, v in user.__dict__.items()} + new_row = self.dict_to_row( + headers, user_dict, to_str=True, info=info, is_valid=str(result), user_id=user.ID + ) + worksheet.append_row(new_row) + + def add_new_student_result_summary( + self, user: GitHubUser, result: PytestResult, info: str = "" + ) -> None: + worksheet = self.summary_spreadsheet.worksheet( + GDRIVE_SUMMARY_SPREADSHEET["worksheets"]["student_challenge_results"]["name"] + ) + headers = worksheet.row_values(1) + result_dict = {k.lower(): v for k, v in result.__dict__.items()} + user_dict = {k.lower(): v for k, v in user.__dict__.items()} + data = {**user_dict, **result_dict} + + new_row = self.dict_to_row(headers, data, to_str=True, info=info) + worksheet.append_row(new_row) + + def add_new_student_detail_results( + self, user: GitHubUser, results: List[Dict[str, Any]], workflow_run_id: int + ) -> None: + + # All worksheets + list_worksheet = self.detail_spreadsheet.worksheets() + # Get student worksheet + student_worksheet = None + for worksheet in list_worksheet: + if worksheet.title == user.LOGIN: + student_worksheet = worksheet + break + + # Create new worksheet + if not student_worksheet: + student_worksheet = self.detail_spreadsheet.add_worksheet( + title=user.LOGIN, rows=1, cols=1 + ) + student_worksheet.insert_row(self.gsheet_details_file.HEADERS) + + headers = student_worksheet.row_values(1) + user_dict = {k.lower(): v for k, v in user.__dict__.items()} + new_rows = [] + + for test in results: + test = {k.lower(): v for k, v in test.items()} + data = {**user_dict, **test} + row = self.dict_to_row(headers, data, to_str=True, workflow_run_id=workflow_run_id) + new_rows.append(row) + self.detail_spreadsheet.values_append( + student_worksheet.title, {"valueInputOption": "USER_ENTERED"}, {"values": new_rows} + ) diff --git a/github_tests_validator_app/lib/models/file.py b/github_tests_validator_app/lib/models/file.py new file mode 100644 index 0000000..2ff5711 --- /dev/null +++ b/github_tests_validator_app/lib/models/file.py @@ -0,0 +1,28 @@ +from typing import List + +from dataclasses import dataclass, field + + +@dataclass +class File: + + NAME: str = "" + ID: str = "" + MIMETYPE: str = "" + + +@dataclass +class WorkSheetFile(File): + HEADERS: str = "" + + +@dataclass +class GSheetFile(File): + + WORKSHEETS: List[WorkSheetFile] = field(default_factory=List[WorkSheetFile]) + + +@dataclass +class GSheetDetailFile(File): + + HEADERS: List[str] = field(default_factory=List[str]) diff --git a/github_tests_validator_app/lib/models/pytest_result.py b/github_tests_validator_app/lib/models/pytest_result.py new file mode 100644 index 0000000..42c57a0 --- /dev/null +++ b/github_tests_validator_app/lib/models/pytest_result.py @@ -0,0 +1,15 @@ +from typing import Any, Dict, Union + +from dataclasses import dataclass, field + + +@dataclass +class PytestResult: + + DURATION: float = 0.0 + TOTAL_TESTS_COLLECTED: int = 0 + TOTAL_PASSED_TEST: int = 0 + TOTAL_FAILED_TEST: int = 0 + WORKFLOW_RUN_ID: int = 0 + DESCRIPTION_TEST_RESULTS: Dict[str, Any] = field(default_factory=Dict[str, Any]) + RESULT: Union[float, None] = None diff --git a/github_tests_validator_app/lib/models/users.py b/github_tests_validator_app/lib/models/users.py new file mode 100644 index 0000000..19720d2 --- /dev/null +++ b/github_tests_validator_app/lib/models/users.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass + + +@dataclass +class GitHubUser: + + LOGIN: str = "" + URL: str = "" + ID: str = "" + CREATED_AT: str = "" diff --git a/github_tests_validator_app/lib/pytest_result.py b/github_tests_validator_app/lib/pytest_result.py deleted file mode 100644 index ab1a8e0..0000000 --- a/github_tests_validator_app/lib/pytest_result.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import Any, Dict, Union - -from dataclasses import dataclass - - -@dataclass -class PytestResult: - - DURATION: float - TOTAL_TESTS_COLLECTED: int - TOTAL_PASSED: int - TOTAL_FAILED: int - DESCRIPTION_TEST_RESULTS: Dict[str, Any] - RESULT: Union[float, None] = None diff --git a/github_tests_validator_app/lib/users.py b/github_tests_validator_app/lib/users.py deleted file mode 100644 index 8e0f4c1..0000000 --- a/github_tests_validator_app/lib/users.py +++ /dev/null @@ -1,22 +0,0 @@ -from typing import Union - -from dataclasses import dataclass -from datetime import datetime - -from github_tests_validator_app.config.config import git_integration - - -@dataclass -class GitHubUser: - - LOGIN: str = "" - URL: str = "" - ID: str = "" - ACCESS_TOKEN: Union[str, None] = None - CREATED_AT: str = datetime.now().strftime("%d/%m/%Y %H:%M:%S") - - def get_access_token(self, repo_name: str) -> str: - self.ACCESS_TOKEN = git_integration.get_access_token( - git_integration.get_installation(self.LOGIN, repo_name).id - ).token - return self.ACCESS_TOKEN diff --git a/github_tests_validator_app/lib/utils.py b/github_tests_validator_app/lib/utils.py index b1cc55b..4b738d1 100644 --- a/github_tests_validator_app/lib/utils.py +++ b/github_tests_validator_app/lib/utils.py @@ -1,9 +1,13 @@ from typing import Any, Dict, List, Union import hashlib +import logging +from datetime import datetime from github import ContentFile -from github_tests_validator_app.lib.users import GitHubUser +from github_tests_validator_app.config.config import DATE_FORMAT +from github_tests_validator_app.lib.connectors.gsheet import GSheetConnector +from github_tests_validator_app.lib.models.users import GitHubUser def get_hash_files(contents: List[ContentFile.ContentFile]) -> str: @@ -23,4 +27,4 @@ def init_github_user_from_github_event(data: Dict[str, Any]) -> Union[GitHubUser login = data["repository"]["owner"].get("login", None) id = data["repository"]["owner"].get("id", None) url = data["repository"]["owner"].get("url", None) - return GitHubUser(LOGIN=login, ID=id, URL=url) + return GitHubUser(LOGIN=login, ID=id, URL=url, CREATED_AT=datetime.now().strftime(DATE_FORMAT)) diff --git a/poetry.lock b/poetry.lock index 8559989..670b255 100644 --- a/poetry.lock +++ b/poetry.lock @@ -290,6 +290,40 @@ python-versions = ">=3.7" [package.dependencies] gitdb = ">=4.0.1,<5" +[[package]] +name = "google-api-core" +version = "2.10.0" +description = "Google API client core library" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +google-auth = ">=1.25.0,<3.0dev" +googleapis-common-protos = ">=1.56.2,<2.0dev" +protobuf = ">=3.20.1,<5.0.0dev" +requests = ">=2.18.0,<3.0.0dev" + +[package.extras] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio-status (>=1.33.2,<2.0dev)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] + +[[package]] +name = "google-api-python-client" +version = "2.60.0" +description = "Google API Client Library for Python" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +google-api-core = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev" +google-auth = ">=1.19.0,<3.0.0dev" +google-auth-httplib2 = ">=0.1.0" +httplib2 = ">=0.15.0,<1dev" +uritemplate = ">=3.0.1,<5" + [[package]] name = "google-auth" version = "2.11.0" @@ -310,6 +344,19 @@ enterprise_cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] pyopenssl = ["pyopenssl (>=20.0.0)"] reauth = ["pyu2f (>=0.1.5)"] +[[package]] +name = "google-auth-httplib2" +version = "0.1.0" +description = "Google Authentication Library: httplib2 transport" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +google-auth = "*" +httplib2 = ">=0.15.0" +six = "*" + [[package]] name = "google-auth-oauthlib" version = "0.5.2" @@ -325,6 +372,20 @@ requests-oauthlib = ">=0.7.0" [package.extras] tool = ["click (>=6.0.0)"] +[[package]] +name = "googleapis-common-protos" +version = "1.56.4" +description = "Common protobufs used in Google APIs" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +protobuf = ">=3.15.0,<5.0.0dev" + +[package.extras] +grpc = ["grpcio (>=1.0.0,<2.0.0dev)"] + [[package]] name = "gspread" version = "5.5.0" @@ -345,6 +406,17 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "httplib2" +version = "0.20.4" +description = "A comprehensive HTTP client library." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0.2,<3.0.3 || >3.0.3,<4", markers = "python_version > \"3.0\""} + [[package]] name = "identify" version = "2.5.5" @@ -516,6 +588,14 @@ pyyaml = ">=5.1" toml = "*" virtualenv = ">=20.0.8" +[[package]] +name = "protobuf" +version = "4.21.5" +description = "" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "py" version = "1.11.0" @@ -939,6 +1019,14 @@ dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2 doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)"] test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] +[[package]] +name = "types-pyyaml" +version = "6.0.11" +description = "Typing stubs for PyYAML" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "types-requests" version = "2.28.9" @@ -966,6 +1054,14 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "uritemplate" +version = "4.1.1" +description = "Implementation of RFC 6570 URI Templates" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "urllib3" version = "1.26.12" @@ -1022,7 +1118,7 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "11f574272877e99f874c61dc9e0e01feb578098d3d2addaeb133e8006b4b31a8" +content-hash = "06aa11dbdd9e6e2654bfc500c9a86d76918763d50c02a223f644f04a9e3777e2" [metadata.files] anyio = [ @@ -1220,14 +1316,30 @@ gitpython = [ {file = "GitPython-3.1.27-py3-none-any.whl", hash = "sha256:5b68b000463593e05ff2b261acff0ff0972df8ab1b70d3cdbd41b546c8b8fc3d"}, {file = "GitPython-3.1.27.tar.gz", hash = "sha256:1c885ce809e8ba2d88a29befeb385fcea06338d3640712b59ca623c220bb5704"}, ] +google-api-core = [ + {file = "google-api-core-2.10.0.tar.gz", hash = "sha256:1d053734f14591939e7764e99c31253fed46bf2578da0dcd82821f17a6dd991c"}, + {file = "google_api_core-2.10.0-py3-none-any.whl", hash = "sha256:325529859836a479244b0882c1a77320fd35cb108df2ec1232e3e908ea56eda4"}, +] +google-api-python-client = [ + {file = "google-api-python-client-2.60.0.tar.gz", hash = "sha256:ca770930a1a8fce969ffdc81910844705138e8d05e040ab62d4276a4b1baddea"}, + {file = "google_api_python_client-2.60.0-py2.py3-none-any.whl", hash = "sha256:a992e2e44bbcdac57797156ac75b2e151a9fd1045c319ab1a970d5ec7f37d102"}, +] google-auth = [ {file = "google-auth-2.11.0.tar.gz", hash = "sha256:ed65ecf9f681832298e29328e1ef0a3676e3732b2e56f41532d45f70a22de0fb"}, {file = "google_auth-2.11.0-py2.py3-none-any.whl", hash = "sha256:be62acaae38d0049c21ca90f27a23847245c9f161ff54ede13af2cb6afecbac9"}, ] +google-auth-httplib2 = [ + {file = "google-auth-httplib2-0.1.0.tar.gz", hash = "sha256:a07c39fd632becacd3f07718dfd6021bf396978f03ad3ce4321d060015cc30ac"}, + {file = "google_auth_httplib2-0.1.0-py2.py3-none-any.whl", hash = "sha256:31e49c36c6b5643b57e82617cb3e021e3e1d2df9da63af67252c02fa9c1f4a10"}, +] google-auth-oauthlib = [ {file = "google-auth-oauthlib-0.5.2.tar.gz", hash = "sha256:d5e98a71203330699f92a26bc08847a92e8c3b1b8d82a021f1af34164db143ae"}, {file = "google_auth_oauthlib-0.5.2-py2.py3-none-any.whl", hash = "sha256:6d6161d0ec0a62e2abf2207c6071c117ec5897b300823c4bb2d963ee86e20e4f"}, ] +googleapis-common-protos = [ + {file = "googleapis-common-protos-1.56.4.tar.gz", hash = "sha256:c25873c47279387cfdcbdafa36149887901d36202cb645a0e4f29686bf6e4417"}, + {file = "googleapis_common_protos-1.56.4-py2.py3-none-any.whl", hash = "sha256:8eb2cbc91b69feaf23e32452a7ae60e791e09967d81d4fcc7fc388182d1bd394"}, +] gspread = [ {file = "gspread-5.5.0-py3-none-any.whl", hash = "sha256:787b5fab9dd61a7d6d84af73356d7ff905cd3978438e528dc66dc8a9407fb851"}, {file = "gspread-5.5.0.tar.gz", hash = "sha256:8620e987e5340315f2b8d8d26cf97e4736a84b3325a17c7d9bcff70525dc3003"}, @@ -1236,6 +1348,10 @@ h11 = [ {file = "h11-0.13.0-py3-none-any.whl", hash = "sha256:8ddd78563b633ca55346c8cd41ec0af27d3c79931828beffb46ce70a379e7442"}, {file = "h11-0.13.0.tar.gz", hash = "sha256:70813c1135087a248a4d38cc0e1a0181ffab2188141a93eaf567940c3957ff06"}, ] +httplib2 = [ + {file = "httplib2-0.20.4-py3-none-any.whl", hash = "sha256:8b6a905cb1c79eefd03f8669fd993c36dc341f7c558f056cb5a33b5c2f458543"}, + {file = "httplib2-0.20.4.tar.gz", hash = "sha256:58a98e45b4b1a48273073f905d2961666ecf0fbac4250ea5b47aef259eb5c585"}, +] identify = [ {file = "identify-2.5.5-py2.py3-none-any.whl", hash = "sha256:ef78c0d96098a3b5fe7720be4a97e73f439af7cf088ebf47b620aeaa10fadf97"}, {file = "identify-2.5.5.tar.gz", hash = "sha256:322a5699daecf7c6fd60e68852f36f2ecbb6a36ff6e6e973e0d2bb6fca203ee6"}, @@ -1356,6 +1472,22 @@ pre-commit = [ {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, ] +protobuf = [ + {file = "protobuf-4.21.5-cp310-abi3-win32.whl", hash = "sha256:5310cbe761e87f0c1decce019d23f2101521d4dfff46034f8a12a53546036ec7"}, + {file = "protobuf-4.21.5-cp310-abi3-win_amd64.whl", hash = "sha256:e5c5a2886ae48d22a9d32fbb9b6636a089af3cd26b706750258ce1ca96cc0116"}, + {file = "protobuf-4.21.5-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:ee04f5823ed98bb9a8c3b1dc503c49515e0172650875c3f76e225b223793a1f2"}, + {file = "protobuf-4.21.5-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:b04484d6f42f48c57dd2737a72692f4c6987529cdd148fb5b8e5f616862a2e37"}, + {file = "protobuf-4.21.5-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:5e0b272217aad8971763960238c1a1e6a65d50ef7824e23300da97569a251c55"}, + {file = "protobuf-4.21.5-cp37-cp37m-win32.whl", hash = "sha256:5eb0724615e90075f1d763983e708e1cef08e66b1891d8b8b6c33bc3b2f1a02b"}, + {file = "protobuf-4.21.5-cp37-cp37m-win_amd64.whl", hash = "sha256:011c0f267e85f5d73750b6c25f0155d5db1e9443cd3590ab669a6221dd8fcdb0"}, + {file = "protobuf-4.21.5-cp38-cp38-win32.whl", hash = "sha256:7b6f22463e2d1053d03058b7b4ceca6e4ed4c14f8c286c32824df751137bf8e7"}, + {file = "protobuf-4.21.5-cp38-cp38-win_amd64.whl", hash = "sha256:b52e7a522911a40445a5f588bd5b5e584291bfc5545e09b7060685e4b2ff814f"}, + {file = "protobuf-4.21.5-cp39-cp39-win32.whl", hash = "sha256:a7faa62b183d6a928e3daffd06af843b4287d16ef6e40f331575ecd236a7974d"}, + {file = "protobuf-4.21.5-cp39-cp39-win_amd64.whl", hash = "sha256:5e0ce02418ef03d7657a420ae8fd6fec4995ac713a3cb09164e95f694dbcf085"}, + {file = "protobuf-4.21.5-py2.py3-none-any.whl", hash = "sha256:bf711b451212dc5b0fa45ae7dada07d8e71a4b0ff0bc8e4783ee145f47ac4f82"}, + {file = "protobuf-4.21.5-py3-none-any.whl", hash = "sha256:3ec6f5b37935406bb9df9b277e79f8ed81d697146e07ef2ba8a5a272fb24b2c9"}, + {file = "protobuf-4.21.5.tar.gz", hash = "sha256:eb1106e87e095628e96884a877a51cdb90087106ee693925ec0a300468a9be3a"}, +] py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, @@ -1619,6 +1751,10 @@ typer = [ {file = "typer-0.6.1-py3-none-any.whl", hash = "sha256:54b19e5df18654070a82f8c2aa1da456a4ac16a2a83e6dcd9f170e291c56338e"}, {file = "typer-0.6.1.tar.gz", hash = "sha256:2d5720a5e63f73eaf31edaa15f6ab87f35f0690f8ca233017d7d23d743a91d73"}, ] +types-pyyaml = [ + {file = "types-PyYAML-6.0.11.tar.gz", hash = "sha256:7f7da2fd11e9bc1e5e9eb3ea1be84f4849747017a59fc2eee0ea34ed1147c2e0"}, + {file = "types_PyYAML-6.0.11-py3-none-any.whl", hash = "sha256:8f890028123607379c63550179ddaec4517dc751f4c527a52bb61934bf495989"}, +] types-requests = [ {file = "types-requests-2.28.9.tar.gz", hash = "sha256:feaf581bd580497a47fe845d506fa3b91b484cf706ff27774e87659837de9962"}, {file = "types_requests-2.28.9-py3-none-any.whl", hash = "sha256:86cb66d3de2f53eac5c09adc42cf6547eefbd0c7e1210beca1ee751c35d96083"}, @@ -1631,6 +1767,10 @@ typing-extensions = [ {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] +uritemplate = [ + {file = "uritemplate-4.1.1-py2.py3-none-any.whl", hash = "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e"}, + {file = "uritemplate-4.1.1.tar.gz", hash = "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0"}, +] urllib3 = [ {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, diff --git a/pyproject.toml b/pyproject.toml index 688e0cf..2ea7a0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,10 @@ urllib3 = ">=1.26.5" gspread = "^5.4.0" types-requests = "^2.28.9" pytest-mock = "^3.8.2" +types-PyYAML = "^6.0.11" +google-api-python-client = "^2.60.0" +google-auth-httplib2 = "^0.1.0" +google-auth-oauthlib = "^0.5.2" [tool.poetry.dev-dependencies] darglint = ">=1.8.0" diff --git a/server.py b/server.py index 5a4f9ad..4e729a7 100644 --- a/server.py +++ b/server.py @@ -1,18 +1,24 @@ from typing import Any +import logging +import traceback + import uvicorn from fastapi import FastAPI, Request -from github_tests_validator_app.bin.github_event_process import validator +from github_tests_validator_app.bin.github_event_process import run app = FastAPI() @app.post("/") -async def main(request: Request) -> Any: - payload = await request.json() +async def main(request: Request): - tests_havent_changed = validator(payload) - return tests_havent_changed + try: + payload = await request.json() + run(payload) + except: + formatted_exception = traceback.format_exc() + logging.error(formatted_exception) if __name__ == "__main__": diff --git a/tests/units/test_github_repo_validation.py b/tests/units/test_github_repo_validation.py index 62592b6..a3e32d4 100644 --- a/tests/units/test_github_repo_validation.py +++ b/tests/units/test_github_repo_validation.py @@ -5,8 +5,8 @@ @pytest.mark.parametrize( "payload,expected", [ - ({}, None), - ({"unkown": "unkown"}, None), + ({}, ""), + ({"unkown": "unkown"}, ""), ({"pull_request": "test"}, "pull_request"), ({"pusher": "test"}, "pusher"), ], diff --git a/tests/units/test_utils.py b/tests/units/test_utils.py index 0b6e8b7..fd0d4d1 100644 --- a/tests/units/test_utils.py +++ b/tests/units/test_utils.py @@ -2,7 +2,7 @@ import pytest from github import ContentFile -from github_tests_validator_app.lib.users import GitHubUser +from github_tests_validator_app.lib.models.users import GitHubUser from github_tests_validator_app.lib.utils import get_hash_files, init_github_user_from_github_event