diff --git a/.env.template b/.env.template index d325380..50d9c17 100644 --- a/.env.template +++ b/.env.template @@ -3,7 +3,6 @@ GH_APP_KEY="-----BEGIN RSA PRIVATE KEY----- Private Key data... -----END PRIVATE KEY-----" GH_PAT="" -GDRIVE_MAIN_DIRECTORY_NAME="" -USER_SHARE="" +SQLALCHEMY_URI="sqlite:///database.db" ENV="" # "LOCAL" or "GCP" GH_TESTS_REPO_NAME="" diff --git a/.gitignore b/.gitignore index 580b6d3..673d00b 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ Icon .terraform/ terraform.tfstate terraform.tfstate.backup +terraform.tfstate.*.backup .terraform.lock.hcl # Files that might appear in the root of a volume diff --git a/README.md b/README.md index a8dce58..14f2c82 100644 --- a/README.md +++ b/README.md @@ -65,16 +65,14 @@ With Cloud Run, you have an example terraform configuration [here](https://githu But you can deploy the application on many Serverless Container services on any cloud by making sure that : - The secrets defined in the `.env` file are available for the container at runtime as environment variables - The container can receive HTTP requests -- The container can use a GCP service account to login with the [Python Google Auth client](https://google-auth.readthedocs.io/en/master/) -- The service account is linked to a GCP Project which has the Google Drive API enabled +- The container can login to any data warehouse with a SQLAlchemy Connection URI : [Bigquery](https://googleapis.dev/python/sqlalchemy-bigquery/latest/README.html#usage), [Snowflake](https://docs.snowflake.com/en/user-guide/sqlalchemy.html#connection-parameters), [Redshift](https://aws.amazon.com/fr/blogs/big-data/use-the-amazon-redshift-sqlalchemy-dialect-to-interact-with-amazon-redshift/) ## Environment variables details - GH_APP_ID : Auto-generated ID of the GitHub App you created during the [`Prerequisites`](#prerequisites) step. - GH_APP_KEY : Private Key of the GitHub App you created during the [`Prerequisites`](#prerequisites) step. - GH_PAT : GitHub personal access token [you must create](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) that has access to the GitHub repository containing the tests and the original repository which was forked (both could be the same repository). -- GDRIVE_MAIN_DIRECTORY_NAME : Name of the Google Drive Folder where you want the stats to be sent. -- USER_SHARE : Comma-separated list of emails that have access to this Google Drive Folder. +- SQLALCHEMY_URI : Database URI with [SQLAlchemy format](https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls) - LOGGING : "LOCAL" if you are deploying locally, "GCP" if you are deploying on Google Cloud Run. - GH_TESTS_REPO_NAME : (Optional, only if you are using a git submodule for the tests folder) Name of the repository containing the tests (could be convenient if you have a repository with the exercices, and another one with the solutions and you want to have the same tests in both repositories by providing a submodule defined in a third repository). diff --git a/docs/architecture.png b/docs/architecture.png index c377e27..9d57cc8 100644 Binary files a/docs/architecture.png and b/docs/architecture.png differ diff --git a/examples/cloud_run/deploy.sh b/examples/cloud_run/deploy.sh index e398d7b..b9aa113 100755 --- a/examples/cloud_run/deploy.sh +++ b/examples/cloud_run/deploy.sh @@ -2,17 +2,25 @@ echo "Please specify GCP project ID : " read PROJECT_ID +echo "Please specify GCP region : " +read REGION source .env gcloud config set project $PROJECT_ID gcloud auth application-default login -export TF_project_id=$PROJECT_ID +export TF_VAR_project_id=$PROJECT_ID +export TF_VAR_region=$REGION +export TF_VAR_docker_image="${REGION}-docker.pkg.dev/${PROJECT_ID}/github-app-registry/no_image" terraform -chdir=examples/cloud_run apply -input=true set +o history echo "$GH_APP_ID" | gcloud secrets versions add GH_APP_ID --data-file=- echo "$GH_APP_KEY" | gcloud secrets versions add GH_APP_KEY --data-file=- echo "$GH_PAT" | gcloud secrets versions add GH_PAT --data-file=- echo "$GH_TESTS_REPO_NAME" | gcloud secrets versions add GH_TESTS_REPO_NAME --data-file=- -echo "$GDRIVE_MAIN_DIRECTORY_NAME" | gcloud secrets versions add GDRIVE_MAIN_DIRECTORY_NAME --data-file=- -echo "$USER_SHARE" | gcloud secrets versions add USER_SHARE --data-file=- +echo "$SQLALCHEMY_URI" | gcloud secrets versions add SQLALCHEMY_URI --data-file=- echo "$LOGGING" | gcloud secrets versions add LOGGING --data-file=- set -o history +gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://${REGION}-docker.pkg.dev +docker build -t ${REGION}-docker.pkg.dev/${PROJECT_ID}/github-app-registry/github_tests_validator_app -f ./docker/Dockerfile . +docker push ${REGION}-docker.pkg.dev/${PROJECT_ID}/github-app-registry/github_tests_validator_app +export TF_VAR_docker_image=$(gcloud artifacts docker images list ${REGION}-docker.pkg.dev/${PROJECT_ID}/github-app-registry --filter="package=${REGION}-docker.pkg.dev/${PROJECT_ID}/github-app-registry/github_tests_validator_app" --sort-by="~UPDATE_TIME" --limit=1 --format="value(format("{0}@{1}",package,version))") +terraform -chdir=examples/cloud_run apply -input=true diff --git a/examples/cloud_run/main.tf b/examples/cloud_run/main.tf index e2ff464..ef38e14 100644 --- a/examples/cloud_run/main.tf +++ b/examples/cloud_run/main.tf @@ -8,6 +8,11 @@ variable "region" { description = "GCP region where resources will be deployed" } +variable "docker_image" { + type = string + description = "Docker reference of the image used by Cloud Run" +} + terraform { required_providers { google = { @@ -22,12 +27,6 @@ provider "google" { region = "${var.region}" } -resource "google_project_service" "drive_api_service" { - project = "${var.project_id}" - service = "drive.googleapis.com" - disable_dependent_services = true -} - resource "google_service_account" "service_account" { project = "${var.project_id}" account_id = "github-tests-validator-app" @@ -61,6 +60,24 @@ resource "google_project_iam_binding" "secret_accessor" { ] } +resource "google_project_iam_binding" "bigquery_job_user" { + project = "${var.project_id}" + role = "roles/bigquery.jobUser" + + members = [ + "serviceAccount:github-tests-validator-app@${var.project_id}.iam.gserviceaccount.com", + ] +} + +resource "google_project_iam_binding" "bigquery_data_editor" { + project = "${var.project_id}" + role = "roles/bigquery.dataEditor" + + members = [ + "serviceAccount:github-tests-validator-app@${var.project_id}.iam.gserviceaccount.com", + ] +} + resource "google_artifact_registry_repository" "github_test_validator_app_registry" { location = "${var.region}" repository_id = "github-app-registry" @@ -76,7 +93,7 @@ resource "google_cloud_run_service" "github_test_validator_app" { timeout_seconds = 300 service_account_name = "github-tests-validator-app@${var.project_id}.iam.gserviceaccount.com" containers { - image = "${var.region}-docker.pkg.dev/${var.project_id}/github-app-registry/github_tests_validator_app:latest" + image = "${var.docker_image}" env { name = "GH_APP_ID" value_from { @@ -114,19 +131,10 @@ resource "google_cloud_run_service" "github_test_validator_app" { } } env { - name = "GDRIVE_MAIN_DIRECTORY_NAME" - value_from { - secret_key_ref { - name = "GDRIVE_MAIN_DIRECTORY_NAME" - key = "latest" - } - } - } - env { - name = "USER_SHARE" + name = "SQLALCHEMY_URI" value_from { secret_key_ref { - name = "USER_SHARE" + name = "SQLALCHEMY_URI" key = "latest" } } @@ -210,19 +218,8 @@ resource "google_secret_manager_secret" "GH_TESTS_REPO_NAME" { } } } -resource "google_secret_manager_secret" "GDRIVE_MAIN_DIRECTORY_NAME" { - secret_id = "GDRIVE_MAIN_DIRECTORY_NAME" - - replication { - user_managed { - replicas { - location = "${var.region}" - } - } - } -} -resource "google_secret_manager_secret" "USER_SHARE" { - secret_id = "USER_SHARE" +resource "google_secret_manager_secret" "SQLALCHEMY_URI" { + secret_id = "SQLALCHEMY_URI" replication { user_managed { diff --git a/github_tests_validator_app/bin/github_event_process.py b/github_tests_validator_app/bin/github_event_process.py index a8ee22d..aa958a7 100644 --- a/github_tests_validator_app/bin/github_event_process.py +++ b/github_tests_validator_app/bin/github_event_process.py @@ -4,28 +4,19 @@ from github_tests_validator_app.bin.github_repo_validation import ( get_event, - get_student_github_connector, + get_user_github_connector, validate_github_repo, ) -from github_tests_validator_app.bin.student_challenge_results_validation import ( - send_student_challenge_results, +from github_tests_validator_app.bin.user_pytest_summaries_validation import ( + send_user_pytest_summaries, ) -from github_tests_validator_app.config import ( - GDRIVE_MAIN_DIRECTORY_NAME, - GDRIVE_SUMMARY_SPREADSHEET, - GSHEET_DETAILS_SPREADSHEET, - USER_SHARE, -) -from github_tests_validator_app.lib.connectors.google_drive import GoogleDriveConnector -from github_tests_validator_app.lib.connectors.google_sheet import GSheetConnector -from github_tests_validator_app.lib.models.file import GSheetDetailFile, GSheetFile, WorkSheetFile -from github_tests_validator_app.lib.models.users import GitHubUser +from github_tests_validator_app.lib.connectors.sqlalchemy_client import SQLAlchemyConnector, User from github_tests_validator_app.lib.utils import init_github_user_from_github_event process = { "pull_request": validate_github_repo, "pusher": validate_github_repo, - "workflow_job": send_student_challenge_results, + "workflow_job": send_user_pytest_summaries, } @@ -35,51 +26,12 @@ def handle_process(payload: Dict[str, Any]) -> str: if ( not event or (event == "pull_request" and payload["action"] not in ["reopened", "opened"]) - or ( - event == "workflow_job" - and ( - payload["action"] not in ["completed"] - or payload["workflow_job"]["conclusion"] != "success" - ) - ) + or (event == "workflow_job" and payload["action"] not in ["completed"]) ): return "" return event -def init_gsheet_file( - google_drive: GoogleDriveConnector, - info: Dict[str, Any], - parent_id: str, - shared_user_list: List[str], -) -> GSheetFile: - - gsheet = google_drive.get_gsheet(info["name"], parent_id, shared_user_list) - list_worksheets = [ - WorkSheetFile(NAME=worksheet["name"], HEADERS=worksheet["headers"]) - for _, worksheet in info["worksheets"].items() - ] - return GSheetFile( - NAME=info["name"], - MIMETYPE=gsheet.get("mimeType", ""), - ID=gsheet.get("id", ""), - WORKSHEETS=list_worksheets, - ) - - -def init_gsheet_detail_file( - google_drive: GoogleDriveConnector, info: Dict[str, Any], parent_id: str, user_share: List[str] -) -> GSheetDetailFile: - - gsheet = google_drive.get_gsheet(info["name"], parent_id, user_share) - return GSheetDetailFile( - NAME=info["name"], - MIMETYPE=gsheet.get("mimeType", ""), - ID=gsheet.get("id", ""), - HEADERS=info["headers"], - ) - - def run(payload: Dict[str, Any]) -> None: """ Validator function @@ -95,42 +47,30 @@ def run(payload: Dict[str, Any]) -> None: if not event: return - # Init Google Drive connector and folders - google_drive = GoogleDriveConnector() - folder = google_drive.get_gdrive_folder(GDRIVE_MAIN_DIRECTORY_NAME, USER_SHARE) - - # Init Google sheets - gsheet_summary_file = init_gsheet_file( - google_drive, GDRIVE_SUMMARY_SPREADSHEET, folder["id"], USER_SHARE - ) - gsheet_details_file = init_gsheet_detail_file( - google_drive, GSHEET_DETAILS_SPREADSHEET, folder["id"], USER_SHARE - ) - - # Init Google sheet connector and worksheets - gsheet = GSheetConnector(google_drive.credentials, gsheet_summary_file, gsheet_details_file) - - # Init GitHubUser - student_user = init_github_user_from_github_event(payload) - if not isinstance(student_user, GitHubUser): + # Init User + user_data = init_github_user_from_github_event(payload) + if not isinstance(user_data, dict): # Logging return - # Send user on Google Sheet - gsheet.add_new_user_on_sheet(student_user) + sql_client = SQLAlchemyConnector() + + sql_client.add_new_user(user_data) # Check valid repo - student_github_connector = get_student_github_connector(student_user, payload) - if not student_github_connector: - gsheet.add_new_repo_valid_result( - student_user, + user_github_connector = get_user_github_connector(user_data, payload) + if not user_github_connector: + sql_client.add_new_repository_validation( + user_data, False, - "[ERROR]: cannot get the student github repository.", + payload, + event, + "[ERROR]: cannot get the user github repository.", ) - logging.error("[ERROR]: cannot get the student github repository.") + logging.error("[ERROR]: cannot get the user github repository.") return logging.info(f'Begin process: "{event}"...') # Run the process - process[event](student_github_connector, gsheet, payload) + process[event](user_github_connector, sql_client, payload, event) logging.info(f'End of process: "{event}".') diff --git a/github_tests_validator_app/bin/github_repo_validation.py b/github_tests_validator_app/bin/github_repo_validation.py index 7ac7f42..da8f96c 100644 --- a/github_tests_validator_app/bin/github_repo_validation.py +++ b/github_tests_validator_app/bin/github_repo_validation.py @@ -8,37 +8,31 @@ GH_TESTS_FOLDER_NAME, GH_TESTS_REPO_NAME, GH_WORKFLOWS_FOLDER_NAME, + commit_ref_path, default_message, ) -from github_tests_validator_app.lib.connectors.github_connector import GitHubConnector -from github_tests_validator_app.lib.connectors.google_sheet import GSheetConnector -from github_tests_validator_app.lib.models.users import GitHubUser - -commit_sha_path: Dict[str, List[str]] = { - "pull_request": ["pull_request", "head", "ref"], - "pusher": ["ref"], - "workflow_job": [], -} +from github_tests_validator_app.lib.connectors.github_client import GitHubConnector +from github_tests_validator_app.lib.connectors.sqlalchemy_client import SQLAlchemyConnector, User def get_event(payload: Dict[str, Any]) -> str: - for event in commit_sha_path: + for event in commit_ref_path: if event in payload: - return event + return str(event) return "" -def get_student_branch(payload: Dict[str, Any], trigger: Union[str, None] = None) -> Any: +def get_user_branch(payload: Dict[str, Any], trigger: Union[str, None] = None) -> Any: trigger = get_event(payload) if not trigger else trigger if not trigger: # Log error # FIXME # Archive the payload # FIXME - logging.error("Couldn't find the student branch, maybe the trigger is not managed") + logging.error("Couldn't find the user branch, maybe the trigger is not managed") return None - path = commit_sha_path[trigger].copy() + path = commit_ref_path[trigger].copy() branch = payload while path: @@ -54,118 +48,139 @@ def get_student_branch(payload: Dict[str, Any], trigger: Union[str, None] = None return branch -def get_student_github_connector( - student: GitHubUser, payload: Dict[str, Any] +def get_user_github_connector( + user_data: Dict[str, Any], payload: Dict[str, Any] ) -> Union[GitHubConnector, None]: - if not student: + if not user_data: return None - github_student_branch = get_student_branch(payload) - if github_student_branch is None: + github_user_branch = get_user_branch(payload) + if github_user_branch is None: return None - return GitHubConnector(student, payload["repository"]["full_name"], github_student_branch) + return GitHubConnector(user_data, payload["repository"]["full_name"], github_user_branch) def compare_folder( - student_github: GitHubConnector, solution_repo: GitHubConnector, folder: str + user_github: GitHubConnector, solution_repo: GitHubConnector, folder: str ) -> Any: - student_contents = student_github.repo.get_contents(folder, ref=student_github.BRANCH_NAME) + user_contents = user_github.repo.get_contents(folder, ref=user_github.BRANCH_NAME) - if ( - isinstance(student_contents, ContentFile.ContentFile) - and student_contents.type == "submodule" - ): + if isinstance(user_contents, ContentFile.ContentFile) and user_contents.type == "submodule": solution_last_commit = solution_repo.get_last_hash_commit() - student_commit = student_contents.sha - return solution_last_commit == student_commit + user_commit = user_contents.sha + return solution_last_commit == user_commit - student_hash = student_github.get_hash(folder) + user_hash = user_github.get_hash(folder) solution_hash = solution_repo.get_hash(folder) - return student_hash == solution_hash + return user_hash == solution_hash def validate_github_repo( - student_github_connector: GitHubConnector, gsheet: GSheetConnector, payload: Dict[str, Any] + user_github_connector: GitHubConnector, + sql_client: SQLAlchemyConnector, + payload: Dict[str, Any], + event: str, ) -> None: logging.info(f"Connecting to repo : {GH_TESTS_REPO_NAME}") tests_github_connector = GitHubConnector( - user=student_github_connector.user, + user_data=user_github_connector.user_data, repo_name=GH_TESTS_REPO_NAME if GH_TESTS_REPO_NAME - else student_github_connector.repo.parent.full_name, + else user_github_connector.repo.parent.full_name, branch_name="main", access_token=GH_PAT, ) - logging.info(f"Connecting to repo : {student_github_connector.repo.parent.full_name}") + logging.info(f"Connecting to repo : {user_github_connector.repo.parent.full_name}") original_github_connector = GitHubConnector( - user=student_github_connector.user, - repo_name=student_github_connector.repo.parent.full_name, + user_data=user_github_connector.user_data, + repo_name=user_github_connector.repo.parent.full_name, branch_name="main", access_token=GH_PAT, ) if not tests_github_connector: - gsheet.add_new_repo_valid_result( - student_github_connector.user, + sql_client.add_new_repository_validation( + user_github_connector.user_data, False, + payload, + event, "[ERROR]: cannot get the tests github repository.", ) logging.error("[ERROR]: cannot get the tests github repository.") return if not original_github_connector: - gsheet.add_new_repo_valid_result( - student_github_connector.user, + sql_client.add_new_repository_validation( + user_github_connector.user_data, False, + payload, + event, "[ERROR]: cannot get the original github repository.", ) logging.error("[ERROR]: cannot get the original github repository.") return workflows_havent_changed = compare_folder( - student_github_connector, original_github_connector, GH_WORKFLOWS_FOLDER_NAME + user_github_connector, original_github_connector, GH_WORKFLOWS_FOLDER_NAME ) tests_havent_changed = compare_folder( - student_github_connector, tests_github_connector, GH_TESTS_FOLDER_NAME + user_github_connector, tests_github_connector, GH_TESTS_FOLDER_NAME ) # Add valid repo result on Google Sheet - gsheet.add_new_repo_valid_result( - student_github_connector.user, + sql_client.add_new_repository_validation( + user_github_connector.user_data, workflows_havent_changed, + payload, + event, default_message["valid_repository"]["workflows"][str(workflows_havent_changed)], ) - gsheet.add_new_repo_valid_result( - student_github_connector.user, + sql_client.add_new_repository_validation( + user_github_connector.user_data, tests_havent_changed, + payload, + event, default_message["valid_repository"]["tests"][str(tests_havent_changed)], ) - # Update Pull Request - if "pull_request" in payload: - issue = student_github_connector.repo.get_issue(number=payload["pull_request"]["number"]) - tests_conclusion = "success" if tests_havent_changed else "failure" - tests_message = default_message["valid_repository"]["tests"][str(tests_havent_changed)] + tests_conclusion = "success" if tests_havent_changed else "failure" + tests_message = default_message["valid_repository"]["tests"][str(tests_havent_changed)] + workflows_conclusion = "success" if workflows_havent_changed else "failure" + workflows_message = default_message["valid_repository"]["workflows"][ + str(workflows_havent_changed) + ] + + if event == "pull_request": + issue = user_github_connector.repo.get_issue(number=payload["pull_request"]["number"]) issue.create_comment(tests_message) - student_github_connector.repo.create_check_run( + user_github_connector.repo.create_check_run( name=tests_message, head_sha=payload["pull_request"]["head"]["sha"], status="completed", conclusion=tests_conclusion, ) - workflows_conclusion = "success" if workflows_havent_changed else "failure" - workflows_message = default_message["valid_repository"]["workflows"][ - str(workflows_havent_changed) - ] - student_github_connector.repo.create_check_run( + user_github_connector.repo.create_check_run( name=workflows_message, head_sha=payload["pull_request"]["head"]["sha"], status="completed", conclusion=workflows_conclusion, ) issue.create_comment(workflows_message) + elif event == "pusher": + user_github_connector.repo.create_check_run( + name=tests_message, + head_sha=payload["after"], + status="completed", + conclusion=tests_conclusion, + ) + user_github_connector.repo.create_check_run( + name=workflows_message, + head_sha=payload["after"], + status="completed", + conclusion=workflows_conclusion, + ) diff --git a/github_tests_validator_app/bin/student_challenge_results_validation.py b/github_tests_validator_app/bin/student_challenge_results_validation.py deleted file mode 100644 index 49ad7e6..0000000 --- a/github_tests_validator_app/bin/student_challenge_results_validation.py +++ /dev/null @@ -1,142 +0,0 @@ -from typing import Any, Dict, List, Tuple, Union - -import logging - -from github_tests_validator_app.config import CHALLENGE_DIR -from github_tests_validator_app.lib.connectors.github_connector import GitHubConnector -from github_tests_validator_app.lib.connectors.google_sheet import GSheetConnector -from github_tests_validator_app.lib.models.pytest_result import PytestResult - - -def init_pytest_result_from_artifact( - artifact: Dict[str, Any], workflow_run_id: int -) -> Union[PytestResult, None]: - if not artifact: - return None - - return PytestResult( - DURATION=artifact["duration"], - TOTAL_TESTS_COLLECTED=artifact["summary"]["collected"], - TOTAL_PASSED_TEST=artifact["summary"]["passed"], - TOTAL_FAILED_TEST=artifact["summary"]["failed"], - DESCRIPTION_TEST_RESULTS=artifact["tests"], - WORKFLOW_RUN_ID=workflow_run_id, - ) - - -def get_student_artifact( - student_github_connector: GitHubConnector, - gsheet: GSheetConnector, - all_student_artifact: Dict[str, Any], - payload: Dict[str, Any], -) -> Any: - - workflow_run_id = payload["workflow_job"]["run_id"] - artifact_info = student_github_connector.get_artifact_info_from_artifacts_with_worflow_run_id( - all_student_artifact["artifacts"], workflow_run_id - ) - if not artifact_info: - gsheet.add_new_student_result_summary( - user=student_github_connector.user, - result=PytestResult(), - info="[ERROR]: Cannot find the artifact of Pytest result on GitHub user repository.", - ) - logging.error( - "[ERROR]: Cannot find the artifact of Pytest result on GitHub user repository." - ) - return None - - # Read Artifact - artifact_resp = student_github_connector.get_artifact(artifact_info) - artifact = student_github_connector.get_artifact_from_format_zip_bytes(artifact_resp.content) - if not artifact: - gsheet.add_new_student_result_summary( - user=student_github_connector.user, - result=PytestResult(), - info="[ERROR]: Cannot read the artifact of Pytest result on GitHub user repository.", - ) - logging.error( - "[ERROR]: Cannot read the artifact of Pytest result on GitHub user repository." - ) - return None - - return artifact - - -def get_test_information(path: str) -> Tuple[str, str, str, str]: - - list_path_name = path.split("::") - file_path = list_path_name[0] - script_name = list_path_name[0].split("/")[-1] - test_name = list_path_name[1] - challenge_id = "-".join( - [ - name[:2] - for name in list_path_name[0].split(CHALLENGE_DIR)[1].split("/") - if ".py" not in name - ] - ) - return challenge_id, file_path, script_name, test_name - - -def parsing_challenge_results(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - - challenge_results = [] - for test in results: - challenge_id, file_path, script_name, test_name = get_test_information(test["nodeid"]) - challenge_results.append( - { - "file_path": file_path, - "script_name": script_name, - "test_name": test_name, - "challenge_id": challenge_id, - "outcome": test["outcome"], - "setup": test["setup"], - "call": test["call"], - "teardown": test["teardown"], - } - ) - - return challenge_results - - -def send_student_challenge_results( - student_github_connector: GitHubConnector, gsheet: GSheetConnector, payload: Dict[str, Any] -) -> None: - - # Get all artifacts - all_student_artifact = student_github_connector.get_all_artifacts() - if not all_student_artifact: - message = f"[ERROR]: Cannot get all artifact on repository {student_github_connector.REPO_NAME} of user {student_github_connector.user.LOGIN}." - if all_student_artifact["total_count"] == 0: - message = f"[ERROR]: No artifact on repository {student_github_connector.REPO_NAME} of user {student_github_connector.user.LOGIN}." - gsheet.add_new_student_result_summary( - user=student_github_connector.user, - result={}, - info=message, - ) - logging.error(message) - return - - # Get student artifact - artifact = get_student_artifact(student_github_connector, gsheet, all_student_artifact, payload) - if not artifact: - return - - # Get summary student results - pytest_result = init_pytest_result_from_artifact(artifact, payload["workflow_job"]["run_id"]) - # Send summary student results to Google Sheet - gsheet.add_new_student_result_summary( - user=student_github_connector.user, - result=pytest_result, - info="Result of student tests", - ) - - # Parsing artifact / challenge results - challenge_results = parsing_challenge_results(artifact["tests"]) - # Send new detail results to Google Sheet - gsheet.add_new_student_detail_results( - user=student_github_connector.user, - results=challenge_results, - workflow_run_id=payload["workflow_job"]["run_id"], - ) diff --git a/github_tests_validator_app/bin/user_pytest_summaries_validation.py b/github_tests_validator_app/bin/user_pytest_summaries_validation.py new file mode 100644 index 0000000..883fcbf --- /dev/null +++ b/github_tests_validator_app/bin/user_pytest_summaries_validation.py @@ -0,0 +1,131 @@ +from typing import Any, Dict, List, Tuple, Union + +import logging +from datetime import datetime + +from github_tests_validator_app.lib.connectors.github_client import GitHubConnector +from github_tests_validator_app.lib.connectors.sqlalchemy_client import SQLAlchemyConnector + + +def get_user_artifact( + user_github_connector: GitHubConnector, + sql_client: SQLAlchemyConnector, + all_user_artifact: Dict[str, Any], + payload: Dict[str, Any], +) -> Any: + + workflow_run_id = payload["workflow_job"]["run_id"] + artifact_info = user_github_connector.get_artifact_info_from_artifacts_with_worflow_run_id( + all_user_artifact["artifacts"], workflow_run_id + ) + if not artifact_info: + sql_client.add_new_pytest_summary( + {}, + workflow_run_id, + user_github_connector.user_data, + user_github_connector.REPO_NAME, + user_github_connector.BRANCH_NAME, + info="[ERROR]: Cannot find the artifact of Pytest result on GitHub user repository.", + ) + logging.error( + "[ERROR]: Cannot find the artifact of Pytest result on GitHub user repository." + ) + return None + + # Read Artifact + artifact_resp = user_github_connector.get_artifact(artifact_info) + artifact = user_github_connector.get_artifact_from_format_zip_bytes(artifact_resp.content) + if not artifact: + sql_client.add_new_pytest_summary( + {}, + workflow_run_id, + user_github_connector.user_data, + user_github_connector.REPO_NAME, + user_github_connector.BRANCH_NAME, + info="[ERROR]: Cannot read the artifact of Pytest result on GitHub user repository.", + ) + logging.error( + "[ERROR]: Cannot read the artifact of Pytest result on GitHub user repository." + ) + return None + + return artifact + + +def get_test_information(path: str) -> Tuple[str, str, str]: + + list_path_name = path.split("::") + file_path = list_path_name[0] + script_name = list_path_name[0].split("/")[-1] + test_name = list_path_name[1] + return file_path, script_name, test_name + + +def parsing_pytest_summaries(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + + pytest_summaries = [] + for test in results: + file_path, script_name, test_name = get_test_information(test["nodeid"]) + pytest_summaries.append( + { + "file_path": file_path, + "script_name": script_name, + "test_name": test_name, + "outcome": test["outcome"], + "setup": test["setup"], + "call": test["call"], + "teardown": test["teardown"], + } + ) + + return pytest_summaries + + +def send_user_pytest_summaries( + user_github_connector: GitHubConnector, + sql_client: SQLAlchemyConnector, + payload: Dict[str, Any], + _: str, +) -> None: + + # Get all artifacts + all_user_artifact = user_github_connector.get_all_artifacts() + if not all_user_artifact: + message = f"[ERROR]: Cannot get all artifact on repository {user_github_connector.REPO_NAME} of user {user_github_connector.organization_or_user}." + if all_user_artifact["total_count"] == 0: + message = f"[ERROR]: No artifact on repository {user_github_connector.REPO_NAME} of user {user_github_connector.organization_or_user}." + sql_client.add_new_pytest_summary( + {}, + payload["workflow_job"]["run_id"], + user_github_connector.user_data, + user_github_connector.REPO_NAME, + user_github_connector.BRANCH_NAME, + info=message, + ) + logging.error(message) + return + + # Get user artifact + artifact = get_user_artifact(user_github_connector, sql_client, all_user_artifact, payload) + if not artifact: + return + + # Send summary user results to Google Sheet + sql_client.add_new_pytest_summary( + artifact, + payload["workflow_job"]["run_id"], + user_github_connector.user_data, + user_github_connector.REPO_NAME, + user_github_connector.BRANCH_NAME, + info="Result of user tests", + ) + + # Parsing artifact / challenge results + pytest_summaries = parsing_pytest_summaries(artifact["tests"]) + # Send new detail results to Google Sheet + sql_client.add_new_pytest_detail( + repository=user_github_connector.REPO_NAME, + branch=user_github_connector.BRANCH_NAME, + results=pytest_summaries, + workflow_run_id=payload["workflow_job"]["run_id"], + ) diff --git a/github_tests_validator_app/config.py b/github_tests_validator_app/config.py index a366e49..de7c73b 100644 --- a/github_tests_validator_app/config.py +++ b/github_tests_validator_app/config.py @@ -1,12 +1,14 @@ -from typing import Dict, cast +from typing import Dict, List, cast import logging import os import google.cloud.logging -import yaml +from dotenv import load_dotenv -if os.getenv("LOGGING", "").replace("\r\n", "") == "GCP": +load_dotenv() + +if os.getenv("LOGGING", "").replace("\r\n", "").replace("\r", "") == "GCP": logging_client = google.cloud.logging.Client() logging_client.get_default_handler() logging_client.setup_logging() @@ -22,29 +24,26 @@ if logging.getLogger("uvicorn") and logging.getLogger("uvicorn").handlers: logging.getLogger("uvicorn").removeHandler(logging.getLogger("uvicorn").handlers[0]) + +commit_ref_path: Dict[str, List[str]] = { + "pull_request": ["pull_request", "head", "ref"], + "pusher": ["ref"], + "workflow_job": [], +} + # GitHub -GH_APP_ID = cast(str, os.getenv("GH_APP_ID", "")).replace("\r\n", "") +GH_APP_ID = cast(str, os.getenv("GH_APP_ID", "")).replace("\r\n", "").replace("\r", "") GH_APP_KEY = cast(str, os.getenv("GH_APP_KEY", "")) -GH_PAT = cast(str, os.getenv("GH_PAT", "")).replace("\r\n", "") -GH_TESTS_REPO_NAME = cast(str, os.getenv("GH_TESTS_REPO_NAME", "")).replace("\r\n", "") +GH_PAT = cast(str, os.getenv("GH_PAT", "")).replace("\r\n", "").replace("\r", "") +SQLALCHEMY_URI = cast(str, os.getenv("SQLALCHEMY_URI", "")).replace("\r\n", "").replace("\r", "") +GH_TESTS_REPO_NAME = ( + cast(str, os.getenv("GH_TESTS_REPO_NAME", "")).replace("\r\n", "").replace("\r", "") +) GH_TESTS_FOLDER_NAME = "tests" GH_WORKFLOWS_FOLDER_NAME = ".github/workflows" GH_API = "https://api.github.com/repos" GH_ALL_ARTIFACT_ENDPOINT = "actions/artifacts" -# Google Drive -GDRIVE_MAIN_DIRECTORY_NAME = cast(str, os.getenv("GDRIVE_MAIN_DIRECTORY_NAME", "")).replace( - "\r\n", "" -) - -# Google Sheet -GDRIVE_HIERARCHY_PATH = "github_tests_validator_app/data/gdrive_hierarchy.yml" -with open(GDRIVE_HIERARCHY_PATH) as file: - data = yaml.safe_load(file) - -GDRIVE_SUMMARY_SPREADSHEET = data["gdrive_summary_spreadsheet"] -GSHEET_DETAILS_SPREADSHEET = data["gsheet_details_spreadsheet"] - # Log message default_message: Dict[str, Dict[str, Dict[str, str]]] = { "valid_repository": { @@ -58,8 +57,3 @@ }, }, } - -# Common -CHALLENGE_DIR = "tests/tests/" -DATE_FORMAT = "%d/%m/%Y %H:%M:%S" -USER_SHARE = os.getenv("USER_SHARE", "").replace("\r\n", "").split(",") diff --git a/github_tests_validator_app/data/gdrive_hierarchy.yml b/github_tests_validator_app/data/gdrive_hierarchy.yml deleted file mode 100644 index 70ddedc..0000000 --- a/github_tests_validator_app/data/gdrive_hierarchy.yml +++ /dev/null @@ -1,45 +0,0 @@ ---- -gdrive_summary_spreadsheet: - name: "Student challenge results" - worksheets: - student: - name: students - headers: - - login - - url - - id - - created_at - check_validation_repo: - name: check_validation_repo - headers: - - login - - user_id - - is_valid - - created_at - - info - student_challenge_results: - name: student_challenge_results - headers: - - login - - workflow_run_id - - created_at - - total_tests_collected - - total_passed_test - - total_failed_test - - duration - - info - -gsheet_details_spreadsheet: - name: Details - headers: - - login - - workflow_run_id - - created_at - - file_path - - script_name - - test_name - - outcome - - challenge_id - - setup - - call - - teardown diff --git a/github_tests_validator_app/lib/connectors/github_connector.py b/github_tests_validator_app/lib/connectors/github_client.py similarity index 85% rename from github_tests_validator_app/lib/connectors/github_connector.py rename to github_tests_validator_app/lib/connectors/github_client.py index 78cb72a..85d5f28 100644 --- a/github_tests_validator_app/lib/connectors/github_connector.py +++ b/github_tests_validator_app/lib/connectors/github_client.py @@ -13,24 +13,25 @@ GH_APP_ID, GH_APP_KEY, ) -from github_tests_validator_app.lib.models.users import GitHubUser from github_tests_validator_app.lib.utils import get_hash_files class GitHubConnector: def __init__( self, - user: GitHubUser, + user_data: Dict[str, Any], repo_name: str, branch_name: str, access_token: Union[str, None] = None, - ): - self.user = user + ) -> None: + self.user_data = user_data self.REPO_NAME = repo_name self.BRANCH_NAME = branch_name self.ACCESS_TOKEN = access_token - logging.info(f"Connecting to Github with user {self.user.LOGIN} on repo: {repo_name} ...") + logging.info( + f"Connecting to Github with user {self.user_data['organization_or_user']} on repo: {repo_name} ..." + ) if not access_token: self.set_git_integration() self.set_access_token(repo_name) @@ -46,12 +47,14 @@ def set_git_integration(self) -> None: def set_access_token(self, repo_name: str) -> None: self.ACCESS_TOKEN = self.git_integration.get_access_token( - self.git_integration.get_installation(self.user.LOGIN, repo_name).id + installation_id=self.git_integration.get_installation( + repo_name.split("/")[0], repo_name.split("/")[1] + ).id, + user_id=self.user_data["id"], ).token def get_repo(self, repo_name: str) -> Repository.Repository: self.REPO_NAME = repo_name - logging.info(f"Connecting to new repo: {repo_name} with user: {self.user.LOGIN} ...") self.repo = self.connector.get_repo(f"{repo_name}") logging.info("Done.") return self.repo @@ -77,7 +80,14 @@ def get_hash(self, folder_name: str) -> str: return hash def get_all_artifacts(self) -> Union[requests.models.Response, Any]: - url = "/".join([GH_API, self.user.LOGIN, self.REPO_NAME, GH_ALL_ARTIFACT_ENDPOINT]) + url = "/".join( + [ + GH_API, + self.user_data["organization_or_user"], + self.REPO_NAME, + GH_ALL_ARTIFACT_ENDPOINT, + ] + ) headers = self._get_headers() response = self._request_data(url, headers=headers) return response @@ -102,7 +112,7 @@ def get_artifact(self, artifact_info: Dict[str, Any]) -> Union[requests.models.R url = "/".join( [ GH_API, - self.user.LOGIN, + self.user_data["organization_or_user"], self.REPO_NAME, GH_ALL_ARTIFACT_ENDPOINT, artifact_id, diff --git a/github_tests_validator_app/lib/connectors/google_drive.py b/github_tests_validator_app/lib/connectors/google_drive.py deleted file mode 100644 index fae17d2..0000000 --- a/github_tests_validator_app/lib/connectors/google_drive.py +++ /dev/null @@ -1,190 +0,0 @@ -from typing import Any, Dict, List - -import logging -from readline import set_completion_display_matches_hook - -from google.auth import default -from googleapiclient.discovery import build - - -class GoogleDriveConnector: - def __init__(self): - logging.info(f"Connecting to Google Drive API ...") - self.credentials = self._get_credentials() - self.client = self._get_client() - logging.info(f"Done.") - - def _get_credentials(self): - credentials, _ = default( - scopes=[ - "https://www.googleapis.com/auth/drive", - ] - ) - return credentials - - def _get_client(self): - return build("drive", "v3", credentials=self.credentials) - - def search_folder(self, folder_name: str) -> Any: - """Get all folders from a google drive. - ... - :return: All folders informations. - :rtype: Any - """ - results = ( - self.client.files() - .list( - q=f"name = '{folder_name}' and (mimeType = 'application/vnd.google-apps.folder')", - spaces="drive", - fields="files(id, name, mimeType, permissions)", - ) - .execute() - ) - return results.get("files", []) - - def get_all_file(self, file_name: str, parent_folder_ids: str = "") -> Any: - """Get all files from a folder on Google Drive. - :param parent_folder_ids: Folder ID. - ... - :return: All file informations from a folder. - :rtype: Any - """ - query = "" - if parent_folder_ids: - query = f"name = '{file_name}'" - response = ( - self.client.files() - .list( - q=query, - spaces="drive", - fields="files(id, name, mimeType, permissions)", - pageToken=None, - ) - .execute() - ) - return response.get("files", []) - - def create_folder(self, folder_name: str) -> Any: - """Create a folder in google drive. - :param folder_name: Folder title. - ... - :return: new folder informations - :rtype: Any - """ - # create drive api client - file_metadata = {"name": folder_name, "mimeType": "application/vnd.google-apps.folder"} - - # pylint: disable=maybe-no-member - folder = self.client.files().create(body=file_metadata).execute() - logging.info(f'Folder {folder["name"]} has created with ID: "{folder["id"]}".') - return folder - - def share_file(self, real_file_id: str, user_email: str) -> List[Any]: - """Share the file with new user email. - :param real_file_id: File ID. - :param user_email: email that we want to share with the file. - ... - :return: informations of the folder - :rtype: List[Any] - """ - ids = [] - file_id = real_file_id - - def callback(request_id, response, exception): - if exception: - logging.error(f"Request_Id: {request_id}") - logging.error(exception) - else: - ids.append(response.get("id")) - - logging.info(f"Sharing file {real_file_id} to : {user_email}") - - batch = self.client.new_batch_http_request(callback=callback) - user_permission = {"type": "user", "role": "writer", "emailAddress": user_email} - batch.add( - self.client.permissions().create(fileId=file_id, body=user_permission, fields="id") - ) - batch.execute() - - return ids - - def share_file_from_users(self, file_info: Dict[str, Any], users: List[str] = []) -> None: - if not users: - return - user_shared = [user["emailAddress"] for user in file_info.get("permissions", [])] - new_shared_users = list(set(users) - set(user_shared)) - for user in new_shared_users: - self.share_file(file_info["id"], user) - - def get_gdrive_folder(self, folder_name: str, shared_user_list: List[str] = []) -> Any: - """Get the folder information in google drive. - .. note :: - If the folder doesn't exist, it will create a new one. - :param folder_name: Folder title. - :param user_share: email that we want to share with the folder. - ... - :return: informations of the folder - :rtype: Any - """ - list_folder = self.search_folder(folder_name) - for folder in list_folder: - if folder.get("name", None) == folder_name: - if shared_user_list: - self.share_file_from_users(folder, shared_user_list) - return folder - - folder = self.create_folder(folder_name) - if "id" in folder and shared_user_list: - self.share_file_from_users(folder, shared_user_list) - return folder - - def get_gsheet( - self, gsheet_name: str, parent_folder_ids: str = "", shared_user_list: List[str] = [] - ) -> Any: - """Get the google sheet information. - .. note :: - If the google sheet doesn't exist, it will create a new one. - :param gsheet_name: Google Sheet title. - :param parent_folder_ids: A list of strings of parent folder ids (if any). - :param user_share: email that we want to share with the google sheet. - ... - :return: informations of the google sheet - :rtype: Any - """ - list_file = self.get_all_file(gsheet_name, parent_folder_ids) - for file in list_file: - if file["name"] == gsheet_name and "spreadsheet" in file["mimeType"]: - if shared_user_list: - self.share_file_from_users(file, shared_user_list) - return file - file = self.create_google_file( - gsheet_name, - "application/vnd.google-apps.spreadsheet", - [parent_folder_ids], - ) - if shared_user_list: - self.share_file_from_users(file, shared_user_list) - return file - - def create_google_file( - self, title: str, mimeType: str, parent_folder_ids: List[str] = [] - ) -> Any: - """Create a new file on Google drive. - .. note :: - Created file is not instantly visible in your Drive search and you need to access it by direct link. - :param title: File title - :param parent_folder_ids: A list of strings of parent folder ids (if any). - ... - :return: informations of new file - :rtype: Any - """ - logging.info(f"Creating Sheet {title}") - body: Dict[str, Any] = { - "name": title, - "mimeType": mimeType, - } - if parent_folder_ids: - body["parents"] = parent_folder_ids - req = self.client.files().create(body=body) - new_sheet = req.execute() - return new_sheet diff --git a/github_tests_validator_app/lib/connectors/google_sheet.py b/github_tests_validator_app/lib/connectors/google_sheet.py deleted file mode 100644 index 10a8fb6..0000000 --- a/github_tests_validator_app/lib/connectors/google_sheet.py +++ /dev/null @@ -1,154 +0,0 @@ -from typing import Any, Dict, List - -import json -import logging - -import gspread -from github_tests_validator_app.config import GDRIVE_SUMMARY_SPREADSHEET -from github_tests_validator_app.lib.models.file import GSheetDetailFile, GSheetFile -from github_tests_validator_app.lib.models.pytest_result import PytestResult -from github_tests_validator_app.lib.models.users import GitHubUser -from google.oauth2.service_account import Credentials - - -class GSheetConnector: - def __init__( - self, - credentials: Credentials, - gsheet_summary_file: GSheetFile, - gsheet_details_file: GSheetDetailFile, - ): - self.gsheet_summary_file = gsheet_summary_file - self.gsheet_details_file = gsheet_details_file - - logging.info(f"Connecting to Google Sheet API ...") - self.gs_client = gspread.authorize(credentials) - logging.info("Done.") - - logging.info(f"Init spreadsheet ...") - self.summary_spreadsheet = self.init_spreadsheet(gsheet_summary_file) - self.detail_spreadsheet = self.gs_client.open_by_key(gsheet_details_file.ID) - logging.info(f"Done.") - - def add_worksheet( - self, spreadsheet: gspread.spreadsheet.Spreadsheet, title: str, headers: List[str] - ) -> gspread.worksheet.Worksheet: - - new_worksheet = spreadsheet.add_worksheet(title=title, rows=1, cols=1) - new_worksheet.insert_row(headers) - return new_worksheet - - def init_spreadsheet(self, gsheet_file: GSheetFile) -> gspread.spreadsheet.Spreadsheet: - - spreadsheet = self.gs_client.open_by_key(gsheet_file.ID) - all_worksheets = spreadsheet.worksheets() - all_worksheets_name = [worksheet.title for worksheet in all_worksheets] - - # Init all worksheets - for worksheet in gsheet_file.WORKSHEETS: - - if worksheet.NAME in all_worksheets_name: - continue - - if all_worksheets and all_worksheets[0].title == "Sheet1": - new_worksheet = all_worksheets.pop(0) - new_worksheet.update_title(worksheet.NAME) - new_worksheet.insert_row(worksheet.HEADERS) - else: - self.add_worksheet(spreadsheet, worksheet.NAME, worksheet.HEADERS) - return spreadsheet - - def add_new_user_on_sheet(self, user: GitHubUser) -> None: - # Controle the workseet exist of not - worksheet = self.summary_spreadsheet.worksheet( - GDRIVE_SUMMARY_SPREADSHEET["worksheets"]["student"]["name"] - ) - - # Check is user exist - id_cell = worksheet.find(str(user.ID)) - login_cell = worksheet.find(user.LOGIN) - if id_cell and login_cell and id_cell.row == login_cell.row: - logging.info("User already exist in student worksheet.") - else: - logging.info(f"Add new user {user.LOGIN} in student worksheet ...") - headers = worksheet.row_values(1) - user_dict = user.__dict__ - new_row = [ - user_dict[header.upper()] if header.upper() in user_dict else None - for header in headers - ] - worksheet.append_row(new_row) - logging.info("Done.") - - def dict_to_row( - self, headers: List[str], data: Dict[str, Any], to_str: bool = False, **kwargs: Any - ) -> List[str]: - result = [] - for header in headers: - value: Any = "" - if header in data: - value = data[header] - elif header in kwargs: - value = kwargs[header] - if to_str and isinstance(value, dict): - value = json.dumps(value) - result.append(value) - return result - - def add_new_repo_valid_result(self, user: GitHubUser, result: bool, info: str = "") -> None: - worksheet = self.summary_spreadsheet.worksheet( - GDRIVE_SUMMARY_SPREADSHEET["worksheets"]["check_validation_repo"]["name"] - ) - headers = worksheet.row_values(1) - user_dict = {k.lower(): v for k, v in user.__dict__.items()} - new_row = self.dict_to_row( - headers, user_dict, to_str=True, info=info, is_valid=str(result), user_id=user.ID - ) - worksheet.append_row(new_row) - - def add_new_student_result_summary( - self, user: GitHubUser, result: PytestResult, info: str = "" - ) -> None: - worksheet = self.summary_spreadsheet.worksheet( - GDRIVE_SUMMARY_SPREADSHEET["worksheets"]["student_challenge_results"]["name"] - ) - headers = worksheet.row_values(1) - result_dict = {k.lower(): v for k, v in result.__dict__.items()} - user_dict = {k.lower(): v for k, v in user.__dict__.items()} - data = {**user_dict, **result_dict} - - new_row = self.dict_to_row(headers, data, to_str=True, info=info) - worksheet.append_row(new_row) - - def add_new_student_detail_results( - self, user: GitHubUser, results: List[Dict[str, Any]], workflow_run_id: int - ) -> None: - - # All worksheets - list_worksheet = self.detail_spreadsheet.worksheets() - # Get student worksheet - student_worksheet = None - for worksheet in list_worksheet: - if worksheet.title == user.LOGIN: - student_worksheet = worksheet - break - - # Create new worksheet - if not student_worksheet: - student_worksheet = self.detail_spreadsheet.add_worksheet( - title=user.LOGIN, rows=1, cols=1 - ) - student_worksheet.insert_row(self.gsheet_details_file.HEADERS) - - headers = student_worksheet.row_values(1) - user_dict = {k.lower(): v for k, v in user.__dict__.items()} - new_rows = [] - - for test in results: - test = {k.lower(): v for k, v in test.items()} - data = {**user_dict, **test} - row = self.dict_to_row(headers, data, to_str=True, workflow_run_id=workflow_run_id) - new_rows.append(row) - self.detail_spreadsheet.values_append( - student_worksheet.title, {"valueInputOption": "USER_ENTERED"}, {"values": new_rows} - ) diff --git a/github_tests_validator_app/lib/connectors/sqlalchemy_client.py b/github_tests_validator_app/lib/connectors/sqlalchemy_client.py new file mode 100644 index 0000000..88d26ad --- /dev/null +++ b/github_tests_validator_app/lib/connectors/sqlalchemy_client.py @@ -0,0 +1,152 @@ +from typing import Any, Dict, List, Optional + +import json +import operator +from datetime import datetime +from functools import reduce + +from github_tests_validator_app.config import SQLALCHEMY_URI, commit_ref_path +from sqlmodel import JSON, Column, Field, Relationship, Session, SQLModel, create_engine + + +class User(SQLModel, table=True): + __tablename__ = "user" + __table_args__ = {"extend_existing": True} + + id: int = Field(primary_key=True) + organization_or_user: str + url: str + created_at: datetime = Field(default=datetime.now()) + + +class WorkflowRun(SQLModel, table=True): + __tablename__ = "workflow_run" + __table_args__ = {"extend_existing": True} + + id: int = Field(primary_key=True) + organization_or_user: str + + repository: str + branch: str + created_at: datetime = Field(default=datetime.now()) + total_tests_collected: int + total_passed_test: int + total_failed_test: int + duration: float + info: str + + user_id: int = Field(foreign_key="user.id") + + +class WorkflowRunDetail(SQLModel, table=True): + __tablename__ = "workflow_run_detail" + __table_args__ = {"extend_existing": True} + + created_at: datetime = Field(primary_key=True, default=datetime.now()) + file_path: str = Field(primary_key=True) + test_name: str = Field(primary_key=True) + repository: str + branch: str + script_name: str + outcome: str + setup: str = Field(default="{}") + call: str = Field(default="{}") + teardown: str = Field(default="{}") + + workflow_run_id: int = Field(foreign_key="workflow_run.id") + + +class RepositoryValidation(SQLModel, table=True): + __tablename__ = "repository_validation" + __table_args__ = {"extend_existing": True} + + repository: str = Field(primary_key=True) + branch: str = Field(primary_key=True) + created_at: datetime = Field(primary_key=True, default=datetime.now()) + organization_or_user: str + is_valid: bool + info: str + + user_id: int = Field(foreign_key="user.id") + + +class SQLAlchemyConnector: + def __init__(self) -> None: + self.engine = create_engine(SQLALCHEMY_URI) + SQLModel.metadata.create_all(self.engine) + + def add_new_user(self, user_data: Dict[str, Any]) -> None: + user = User(**user_data) + with Session(self.engine) as session: + session.add(user) + session.commit() + + def add_new_repository_validation( + self, + user_data: Dict[str, Any], + result: bool, + payload: Dict[str, Any], + event: str, + info: str = "", + ) -> None: + repository_validation = RepositoryValidation( + repository=payload["repository"]["full_name"], + branch=reduce(operator.getitem, commit_ref_path[event], payload), + created_at=datetime.now(), + organization_or_user=user_data["organization_or_user"], + user_id=user_data["id"], + is_valid=result, + info=info, + ) + with Session(self.engine) as session: + session.add(repository_validation) + session.commit() + + def add_new_pytest_summary( + self, + artifact: Dict[str, Any], + workflow_run_id: int, + user_data: Dict[str, Any], + repository: str, + branch: str, + info: str, + ) -> None: + pytest_summary = WorkflowRun( + id=workflow_run_id, + organization_or_user=user_data["organization_or_user"], + user_id=user_data["id"], + repository=repository, + branch=branch, + duration=artifact.get("duration", None), + total_tests_collected=artifact.get("summary", {}).get("collected", None), + total_passed_test=artifact.get("summary", {}).get("passed", None), + total_failed_test=artifact.get("summary", {}).get("failed", None), + info=info, + ) + with Session(self.engine) as session: + session.add(pytest_summary) + session.commit() + + def add_new_pytest_detail( + self, + repository: str, + branch: str, + results: List[Dict[str, Any]], + workflow_run_id: int, + ) -> None: + with Session(self.engine) as session: + for test in results: + pytest_detail = WorkflowRunDetail( + repository=repository, + branch=branch, + workflow_run_id=workflow_run_id, + file_path=test["file_path"], + test_name=test["test_name"], + script_name=test["script_name"], + outcome=test["outcome"], + setup=json.dumps(test["setup"]), + call=json.dumps(test["call"]), + teardown=json.dumps(test["teardown"]), + ) + session.add(pytest_detail) + session.commit() diff --git a/github_tests_validator_app/lib/models/file.py b/github_tests_validator_app/lib/models/file.py deleted file mode 100644 index 2ff5711..0000000 --- a/github_tests_validator_app/lib/models/file.py +++ /dev/null @@ -1,28 +0,0 @@ -from typing import List - -from dataclasses import dataclass, field - - -@dataclass -class File: - - NAME: str = "" - ID: str = "" - MIMETYPE: str = "" - - -@dataclass -class WorkSheetFile(File): - HEADERS: str = "" - - -@dataclass -class GSheetFile(File): - - WORKSHEETS: List[WorkSheetFile] = field(default_factory=List[WorkSheetFile]) - - -@dataclass -class GSheetDetailFile(File): - - HEADERS: List[str] = field(default_factory=List[str]) diff --git a/github_tests_validator_app/lib/models/pytest_result.py b/github_tests_validator_app/lib/models/pytest_result.py deleted file mode 100644 index 42c57a0..0000000 --- a/github_tests_validator_app/lib/models/pytest_result.py +++ /dev/null @@ -1,15 +0,0 @@ -from typing import Any, Dict, Union - -from dataclasses import dataclass, field - - -@dataclass -class PytestResult: - - DURATION: float = 0.0 - TOTAL_TESTS_COLLECTED: int = 0 - TOTAL_PASSED_TEST: int = 0 - TOTAL_FAILED_TEST: int = 0 - WORKFLOW_RUN_ID: int = 0 - DESCRIPTION_TEST_RESULTS: Dict[str, Any] = field(default_factory=Dict[str, Any]) - RESULT: Union[float, None] = None diff --git a/github_tests_validator_app/lib/models/users.py b/github_tests_validator_app/lib/models/users.py deleted file mode 100644 index 19720d2..0000000 --- a/github_tests_validator_app/lib/models/users.py +++ /dev/null @@ -1,10 +0,0 @@ -from dataclasses import dataclass - - -@dataclass -class GitHubUser: - - LOGIN: str = "" - URL: str = "" - ID: str = "" - CREATED_AT: str = "" diff --git a/github_tests_validator_app/lib/utils.py b/github_tests_validator_app/lib/utils.py index 7717814..eeff3fd 100644 --- a/github_tests_validator_app/lib/utils.py +++ b/github_tests_validator_app/lib/utils.py @@ -1,13 +1,10 @@ -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Optional import hashlib -import logging from datetime import datetime from github import ContentFile -from github_tests_validator_app.config import DATE_FORMAT -from github_tests_validator_app.lib.connectors.google_sheet import GSheetConnector -from github_tests_validator_app.lib.models.users import GitHubUser +from github_tests_validator_app.lib.connectors.sqlalchemy_client import User def get_hash_files(contents: List[ContentFile.ContentFile]) -> str: @@ -19,12 +16,12 @@ def get_hash_files(contents: List[ContentFile.ContentFile]) -> str: return str(hash.hexdigest()) -def init_github_user_from_github_event(data: Dict[str, Any]) -> Union[GitHubUser, None]: +def init_github_user_from_github_event(data: Dict[str, Any]) -> Optional[Dict[str, Any]]: - if not "repository" in data or not "owner" in data["repository"]: + if not "sender" in data: return None - login = data["repository"]["owner"].get("login", None) - id = data["repository"]["owner"].get("id", None) - url = data["repository"]["owner"].get("url", None) - return GitHubUser(LOGIN=login, ID=id, URL=url, CREATED_AT=datetime.now().strftime(DATE_FORMAT)) + login = data["sender"]["login"] + id = data["sender"]["id"] + url = data["sender"]["url"] + return dict(id=id, organization_or_user=login, url=url, created_at=datetime.now()) diff --git a/poetry.lock b/poetry.lock index 94c9a50..5d47f95 100644 --- a/poetry.lock +++ b/poetry.lock @@ -15,6 +15,14 @@ doc = ["packaging", "sphinx-rtd-theme", "sphinx-autodoc-typehints (>=1.2.0)"] test = ["coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "contextlib2", "uvloop (<0.15)", "mock (>=4)", "uvloop (>=0.15)"] trio = ["trio (>=0.16)"] +[[package]] +name = "asn1crypto" +version = "1.5.1" +description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "astroid" version = "2.12.11" @@ -26,10 +34,7 @@ python-versions = ">=3.7.2" [package.dependencies] lazy-object-proxy = ">=1.4.0" typing-extensions = {version = ">=3.10", markers = "python_version < \"3.10\""} -wrapt = [ - {version = ">=1.11,<2", markers = "python_version < \"3.11\""}, - {version = ">=1.14,<2", markers = "python_version >= \"3.11\""}, -] +wrapt = {version = ">=1.11,<2", markers = "python_version < \"3.11\""} [[package]] name = "attrs" @@ -64,6 +69,21 @@ test = ["coverage (>=4.5.4)", "fixtures (>=3.0.0)", "flake8 (>=4.0.0)", "stestr toml = ["toml"] yaml = ["pyyaml"] +[[package]] +name = "beautifulsoup4" +version = "4.11.1" +description = "Screen-scraping library" +category = "main" +optional = false +python-versions = ">=3.6.0" + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + [[package]] name = "black" version = "22.10.0" @@ -86,6 +106,38 @@ d = ["aiohttp (>=3.7.4)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] +[[package]] +name = "boto3" +version = "1.24.90" +description = "The AWS SDK for Python" +category = "main" +optional = false +python-versions = ">= 3.7" + +[package.dependencies] +botocore = ">=1.27.90,<1.28.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.6.0,<0.7.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.27.90" +description = "Low-level, data-driven core of boto 3." +category = "main" +optional = false +python-versions = ">= 3.7" + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<1.27" + +[package.extras] +crt = ["awscrt (==0.14.0)"] + [[package]] name = "cachetools" version = "5.2.0" @@ -164,7 +216,7 @@ test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] [[package]] name = "cryptography" -version = "38.0.1" +version = "36.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = false @@ -177,9 +229,9 @@ cffi = ">=1.12" docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] docstest = ["pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] -sdist = ["setuptools-rust (>=0.11.4)"] +sdist = ["setuptools_rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] +test = ["pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] [[package]] name = "darglint" @@ -260,7 +312,7 @@ test = ["anyio[trio] (>=3.2.1,<4.0.0)", "black (==22.8.0)", "databases[sqlite] ( name = "filelock" version = "3.8.0" description = "A platform independent file lock." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" @@ -268,6 +320,14 @@ python-versions = ">=3.7" docs = ["furo (>=2022.6.21)", "sphinx (>=5.1.1)", "sphinx-autodoc-typehints (>=1.19.1)"] testing = ["covdefaults (>=2.2)", "coverage (>=6.4.2)", "pytest (>=7.1.2)", "pytest-cov (>=3)", "pytest-timeout (>=2.1)"] +[[package]] +name = "future" +version = "0.18.2" +description = "Clean single-source support for Python 3 and 2" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + [[package]] name = "gitdb" version = "4.0.9" @@ -281,7 +341,7 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.28" +version = "3.1.29" description = "GitPython is a python library used to interact with Git repositories" category = "dev" optional = false @@ -311,21 +371,6 @@ grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio-status (>=1.33.2,<2.0dev)"] grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] -[[package]] -name = "google-api-python-client" -version = "2.64.0" -description = "Google API Client Library for Python" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -google-api-core = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev" -google-auth = ">=1.19.0,<3.0.0dev" -google-auth-httplib2 = ">=0.1.0" -httplib2 = ">=0.15.0,<1dev" -uritemplate = ">=3.0.1,<5" - [[package]] name = "google-auth" version = "2.12.0" @@ -347,57 +392,76 @@ pyopenssl = ["pyopenssl (>=20.0.0)"] reauth = ["pyu2f (>=0.1.5)"] [[package]] -name = "google-auth-httplib2" -version = "0.1.0" -description = "Google Authentication Library: httplib2 transport" +name = "google-cloud-appengine-logging" +version = "1.1.6" +description = "" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.7" [package.dependencies] -google-auth = "*" -httplib2 = ">=0.15.0" -six = "*" +google-api-core = {version = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev", extras = ["grpc"]} +proto-plus = ">=1.22.0,<2.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" [[package]] -name = "google-auth-oauthlib" -version = "0.5.3" -description = "Google Authentication Library" +name = "google-cloud-audit-log" +version = "0.2.4" +description = "Google Cloud Audit Protos" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] -google-auth = ">=1.0.0" -requests-oauthlib = ">=0.7.0" - -[package.extras] -tool = ["click (>=6.0.0)"] +googleapis-common-protos = ">=1.56.2,<2.0dev" +protobuf = ">=3.6.0,<5.0.0dev" [[package]] -name = "google-cloud-appengine-logging" -version = "1.1.5" -description = "" +name = "google-cloud-bigquery" +version = "3.3.5" +description = "Google BigQuery API client library" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.7, <3.11" [package.dependencies] google-api-core = {version = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev", extras = ["grpc"]} +google-cloud-bigquery-storage = ">=2.0.0,<3.0.0dev" +google-cloud-core = ">=1.4.1,<3.0.0dev" +google-resumable-media = ">=0.6.0,<3.0dev" +grpcio = ">=1.47.0,<2.0dev" +packaging = ">=14.3,<22.0.0dev" proto-plus = ">=1.22.0,<2.0.0dev" -protobuf = ">=3.20.2,<5.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" +pyarrow = ">=3.0.0,<10.0dev" +python-dateutil = ">=2.7.2,<3.0dev" +requests = ">=2.18.0,<3.0.0dev" + +[package.extras] +all = ["pandas (>=1.0.0)", "db-dtypes (>=0.3.0,<2.0.0dev)", "geopandas (>=0.9.0,<1.0dev)", "Shapely (>=1.6.0,<2.0dev)", "ipython (>=7.0.1,!=8.1.0)", "tqdm (>=4.7.4,<5.0.0dev)", "opentelemetry-api (>=1.1.0)", "opentelemetry-sdk (>=1.1.0)", "opentelemetry-instrumentation (>=0.20b0)"] +geopandas = ["geopandas (>=0.9.0,<1.0dev)", "Shapely (>=1.6.0,<2.0dev)"] +ipython = ["ipython (>=7.0.1,!=8.1.0)"] +opentelemetry = ["opentelemetry-api (>=1.1.0)", "opentelemetry-sdk (>=1.1.0)", "opentelemetry-instrumentation (>=0.20b0)"] +pandas = ["pandas (>=1.0.0)", "db-dtypes (>=0.3.0,<2.0.0dev)"] +tqdm = ["tqdm (>=4.7.4,<5.0.0dev)"] [[package]] -name = "google-cloud-audit-log" -version = "0.2.4" -description = "Google Cloud Audit Protos" +name = "google-cloud-bigquery-storage" +version = "2.16.2" +description = "BigQuery Storage API API client library" category = "main" optional = false python-versions = ">=3.7" [package.dependencies] -googleapis-common-protos = ">=1.56.2,<2.0dev" -protobuf = ">=3.6.0,<5.0.0dev" +google-api-core = {version = ">=1.32.0,<2.0.0 || >=2.8.0,<3.0.0dev", extras = ["grpc"]} +proto-plus = ">=1.22.0,<2.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" + +[package.extras] +fastavro = ["fastavro (>=0.21.2)"] +pandas = ["pandas (>=0.21.1)"] +pyarrow = ["pyarrow (>=0.15.0)"] [[package]] name = "google-cloud-core" @@ -416,7 +480,7 @@ grpc = ["grpcio (>=1.38.0,<2.0dev)"] [[package]] name = "google-cloud-logging" -version = "3.2.4" +version = "3.2.5" description = "Stackdriver Logging API client library" category = "main" optional = false @@ -429,7 +493,33 @@ google-cloud-audit-log = ">=0.1.0,<1.0.0dev" google-cloud-core = ">=2.0.0,<3.0.0dev" grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" proto-plus = ">=1.22.0,<2.0.0dev" -protobuf = ">=3.20.2,<5.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" + +[[package]] +name = "google-crc32c" +version = "1.5.0" +description = "A python wrapper of the C library 'Google CRC32C'" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +testing = ["pytest"] + +[[package]] +name = "google-resumable-media" +version = "2.4.0" +description = "Utilities for Google Media Downloads and Resumable Uploads" +category = "main" +optional = false +python-versions = ">= 3.7" + +[package.dependencies] +google-crc32c = ">=1.0,<2.0dev" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)"] +requests = ["requests (>=2.18.0,<3.0.0dev)"] [[package]] name = "googleapis-common-protos" @@ -446,6 +536,17 @@ protobuf = ">=3.15.0,<5.0.0dev" [package.extras] grpc = ["grpcio (>=1.0.0,<2.0.0dev)"] +[[package]] +name = "greenlet" +version = "1.1.3.post0" +description = "Lightweight in-process concurrent programming" +category = "main" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" + +[package.extras] +docs = ["sphinx"] + [[package]] name = "grpc-google-iam-v1" version = "0.12.4" @@ -485,18 +586,6 @@ googleapis-common-protos = ">=1.5.5" grpcio = ">=1.49.1" protobuf = ">=4.21.3" -[[package]] -name = "gspread" -version = "5.6.0" -description = "Google Spreadsheets Python API" -category = "main" -optional = false -python-versions = ">=3.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.dependencies] -google-auth = ">=1.12.0" -google-auth-oauthlib = ">=0.4.1" - [[package]] name = "h11" version = "0.14.0" @@ -505,17 +594,6 @@ category = "main" optional = false python-versions = ">=3.7" -[[package]] -name = "httplib2" -version = "0.20.4" -description = "A comprehensive HTTP client library." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.dependencies] -pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0.2,<3.0.3 || >3.0.3,<4", markers = "python_version > \"3.0\""} - [[package]] name = "identify" version = "2.5.6" @@ -557,6 +635,14 @@ requirements_deprecated_finder = ["pipreqs", "pip-api"] colors = ["colorama (>=0.4.3,<0.5.0)"] plugins = ["setuptools"] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "lazy-object-proxy" version = "1.7.1" @@ -565,6 +651,20 @@ category = "dev" optional = false python-versions = ">=3.6" +[[package]] +name = "lxml" +version = "4.9.1" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html5 = ["html5lib"] +htmlsoup = ["beautifulsoup4"] +source = ["Cython (>=0.29.7)"] + [[package]] name = "mccabe" version = "0.7.0" @@ -608,17 +708,23 @@ optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" [[package]] -name = "oauthlib" -version = "3.2.1" -description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +name = "numpy" +version = "1.23.4" +description = "NumPy is the fundamental package for array computing with Python." category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" -[package.extras] -rsa = ["cryptography (>=3.0.0)"] -signals = ["blinker (>=1.4.0)"] -signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] +[[package]] +name = "oscrypto" +version = "1.3.0" +description = "TLS (SSL) sockets, key generation, encryption, decryption, signing, verification and KDFs using the OS crypto libraries. Does not require a compiler, and relies on the OS for patching. Works on Windows, OS X and Linux/BSD." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +asn1crypto = ">=1.5.1" [[package]] name = "packaging" @@ -717,6 +823,17 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "pyarrow" +version = "6.0.1" +description = "Python library for Apache Arrow" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +numpy = ">=1.16.6" + [[package]] name = "pyasn1" version = "0.4.8" @@ -744,6 +861,14 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "pycryptodomex" +version = "3.15.0" +description = "Cryptographic library for Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + [[package]] name = "pydantic" version = "1.10.2" @@ -775,7 +900,7 @@ toml = ["toml"] [[package]] name = "pygithub" -version = "1.55" +version = "1.56" description = "Use the full Github API v3" category = "main" optional = false @@ -853,6 +978,21 @@ cffi = ">=1.4.1" docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] tests = ["pytest (>=3.2.1,!=3.3.0)", "hypothesis (>=3.27.0)"] +[[package]] +name = "pyopenssl" +version = "22.0.0" +description = "Python wrapper module around the OpenSSL library" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cryptography = ">=35.0" + +[package.extras] +docs = ["sphinx", "sphinx-rtd-theme"] +test = ["flaky", "pretend", "pytest (>=3.0.1)"] + [[package]] name = "pyparsing" version = "3.0.9" @@ -898,9 +1038,39 @@ pytest = ">=5.0" [package.extras] dev = ["pre-commit", "tox", "pytest-asyncio"] +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "python-dotenv" +version = "0.21.0" +description = "Read key-value pairs from a .env file and set them as environment variables" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +cli = ["click (>=5.0)"] + +[[package]] +name = "pytz" +version = "2022.4" +description = "World timezone definitions, modern and historical" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "pyupgrade" -version = "3.0.0" +version = "3.1.0" description = "A tool to automatically upgrade syntax for newer versions." category = "dev" optional = false @@ -917,6 +1087,27 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "redshift-connector" +version = "2.0.909" +description = "Redshift interface library" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +beautifulsoup4 = ">=4.7.0,<5.0.0" +boto3 = ">=1.9.201,<2.0.0" +botocore = ">=1.12.201,<2.0.0" +lxml = ">=4.6.5" +packaging = "*" +pytz = ">=2020.1" +requests = ">=2.23.0,<3.0.0" +scramp = ">=1.2.0,<1.5.0" + +[package.extras] +full = ["numpy", "pandas"] + [[package]] name = "requests" version = "2.28.1" @@ -935,21 +1126,6 @@ urllib3 = ">=1.21.1,<1.27" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] -[[package]] -name = "requests-oauthlib" -version = "1.3.1" -description = "OAuthlib authentication support for Requests." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.dependencies] -oauthlib = ">=3.0.0" -requests = ">=2.0.0" - -[package.extras] -rsa = ["oauthlib[signedtoken] (>=3.0.0)"] - [[package]] name = "rich" version = "12.6.0" @@ -999,6 +1175,20 @@ category = "dev" optional = false python-versions = ">=3.5" +[[package]] +name = "s3transfer" +version = "0.6.0" +description = "An Amazon S3 Transfer Manager" +category = "main" +optional = false +python-versions = ">= 3.7" + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + [[package]] name = "safety" version = "2.3.1" @@ -1018,6 +1208,17 @@ requests = "*" github = ["pygithub (>=1.43.3)", "jinja2 (>=3.1.0)"] gitlab = ["python-gitlab (>=1.3.0)"] +[[package]] +name = "scramp" +version = "1.4.1" +description = "An implementation of the SCRAM protocol." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +asn1crypto = ">=1.4.0" + [[package]] name = "shellingham" version = "1.5.0" @@ -1058,6 +1259,151 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "snowflake-connector-python" +version = "2.8.0" +description = "Snowflake Connector for Python" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +asn1crypto = ">0.24.0,<2.0.0" +certifi = ">=2017.4.17" +cffi = ">=1.9,<2.0.0" +charset-normalizer = ">=2,<3" +cryptography = ">=3.1.0,<37.0.0" +filelock = ">=3.5,<4" +idna = ">=2.5,<4" +oscrypto = "<2.0.0" +pycryptodomex = ">=3.2,<3.5.0 || >3.5.0,<4.0.0" +pyjwt = "<3.0.0" +pyOpenSSL = ">=16.2.0,<23.0.0" +pytz = "*" +requests = "<3.0.0" +typing-extensions = ">=4.3,<5" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +development = ["cython", "coverage", "more-itertools", "numpy (<1.24.0)", "pendulum (!=2.1.1)", "pexpect", "pytest (<7.2.0)", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytest-xdist", "pytzdata"] +pandas = ["pandas (>=1.0.0,<1.5.0)", "pyarrow (>=8.0.0,<8.1.0)"] +secure-local-storage = ["keyring (!=16.1.0,<24.0.0)"] + +[[package]] +name = "snowflake-sqlalchemy" +version = "1.4.2" +description = "Snowflake SQLAlchemy Dialect" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +snowflake-connector-python = "<3.0.0" +sqlalchemy = ">=1.4.0,<2.0.0" + +[package.extras] +development = ["pytest", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "mock", "pytz", "numpy"] +pandas = ["snowflake-connector-python[pandas] (<3.0.0)"] + +[[package]] +name = "soupsieve" +version = "2.3.2.post1" +description = "A modern CSS selector implementation for Beautiful Soup." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "sqlalchemy" +version = "1.4.27" +description = "Database Abstraction Library" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" + +[package.dependencies] +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} + +[package.extras] +aiomysql = ["greenlet (!=0.4.17)", "aiomysql"] +aiosqlite = ["typing_extensions (!=3.10.0.1)", "greenlet (!=0.4.17)", "aiosqlite"] +asyncio = ["greenlet (!=0.4.17)"] +asyncmy = ["greenlet (!=0.4.17)", "asyncmy (>=0.2.3)"] +mariadb_connector = ["mariadb (>=1.0.1)"] +mssql = ["pyodbc"] +mssql_pymssql = ["pymssql"] +mssql_pyodbc = ["pyodbc"] +mypy = ["sqlalchemy2-stubs", "mypy (>=0.910)"] +mysql = ["mysqlclient (>=1.4.0,<2)", "mysqlclient (>=1.4.0)"] +mysql_connector = ["mysql-connector-python"] +oracle = ["cx_oracle (>=7,<8)", "cx_oracle (>=7)"] +postgresql = ["psycopg2 (>=2.7)"] +postgresql_asyncpg = ["greenlet (!=0.4.17)", "asyncpg"] +postgresql_pg8000 = ["pg8000 (>=1.16.6)"] +postgresql_psycopg2binary = ["psycopg2-binary"] +postgresql_psycopg2cffi = ["psycopg2cffi"] +pymysql = ["pymysql (<1)", "pymysql"] +sqlcipher = ["sqlcipher3-binary"] + +[[package]] +name = "sqlalchemy-bigquery" +version = "1.4.4" +description = "SQLAlchemy dialect for BigQuery" +category = "main" +optional = false +python-versions = ">=3.6, <3.11" + +[package.dependencies] +future = "*" +google-api-core = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev" +google-auth = ">=1.25.0,<3.0.0dev" +google-cloud-bigquery = ">=2.25.2,<4.0.0dev" +google-cloud-bigquery-storage = ">=2.0.0,<3.0.0dev" +pyarrow = ">=3.0.0,<7.0dev" +sqlalchemy = ">=1.2.0,<=1.4.27" + +[package.extras] +alembic = ["alembic"] +all = ["alembic", "shapely", "pytz", "geoalchemy2", "packaging"] +geography = ["geoalchemy2", "shapely"] +tests = ["packaging", "pytz"] + +[[package]] +name = "sqlalchemy-redshift" +version = "0.8.11" +description = "Amazon Redshift Dialect for sqlalchemy" +category = "main" +optional = false +python-versions = ">=3.4" + +[package.dependencies] +packaging = "*" +SQLAlchemy = ">=0.9.2,<2.0.0" + +[[package]] +name = "sqlalchemy2-stubs" +version = "0.0.2a29" +description = "Typing Stubs for SQLAlchemy 1.4" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +typing-extensions = ">=3.7.4" + +[[package]] +name = "sqlmodel" +version = "0.0.8" +description = "SQLModel, SQL databases in Python, designed for simplicity, compatibility, and robustness." +category = "main" +optional = false +python-versions = ">=3.6.1,<4.0.0" + +[package.dependencies] +pydantic = ">=1.8.2,<2.0.0" +SQLAlchemy = ">=1.4.17,<=1.4.41" +sqlalchemy2-stubs = "*" + [[package]] name = "starlette" version = "0.20.4" @@ -1075,7 +1421,7 @@ full = ["itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests"] [[package]] name = "stevedore" -version = "4.0.0" +version = "4.0.1" description = "Manage dynamic plugins for Python applications" category = "dev" optional = false @@ -1118,7 +1464,7 @@ python-versions = ">=3.6,<4.0" [[package]] name = "typer" -version = "0.6.1" +version = "0.7.0" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." category = "main" optional = false @@ -1131,10 +1477,10 @@ rich = {version = ">=10.11.0,<13.0.0", optional = true, markers = "extra == \"al shellingham = {version = ">=1.3.0,<2.0.0", optional = true, markers = "extra == \"all\""} [package.extras] -test = ["rich (>=10.11.0,<13.0.0)", "isort (>=5.0.6,<6.0.0)", "black (>=22.3.0,<23.0.0)", "mypy (==0.910)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "coverage (>=5.2,<6.0)", "pytest-cov (>=2.10.0,<3.0.0)", "pytest (>=4.4.0,<5.4.0)", "shellingham (>=1.3.0,<2.0.0)"] -doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "mkdocs (>=1.1.2,<2.0.0)"] -dev = ["pre-commit (>=2.17.0,<3.0.0)", "flake8 (>=3.8.3,<4.0.0)", "autoflake (>=1.3.1,<2.0.0)"] -all = ["rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)", "colorama (>=0.4.3,<0.5.0)"] +all = ["colorama (>=0.4.3,<0.5.0)", "shellingham (>=1.3.0,<2.0.0)", "rich (>=10.11.0,<13.0.0)"] +dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] +doc = ["mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "mdx-include (>=1.4.1,<2.0.0)", "pillow (>=9.3.0,<10.0.0)", "cairosvg (>=2.5.2,<3.0.0)"] +test = ["shellingham (>=1.3.0,<2.0.0)", "pytest (>=4.4.0,<8.0.0)", "pytest-cov (>=2.10.0,<5.0.0)", "coverage (>=6.2,<7.0)", "pytest-xdist (>=1.32.0,<4.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "mypy (==0.910)", "black (>=22.3.0,<23.0.0)", "isort (>=5.0.6,<6.0.0)", "rich (>=10.11.0,<13.0.0)"] [[package]] name = "types-pyyaml" @@ -1171,14 +1517,6 @@ category = "main" optional = false python-versions = ">=3.7" -[[package]] -name = "uritemplate" -version = "4.1.1" -description = "Implementation of RFC 6570 URI Templates" -category = "main" -optional = false -python-versions = ">=3.6" - [[package]] name = "urllib3" version = "1.26.12" @@ -1234,18 +1572,25 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [metadata] lock-version = "1.1" -python-versions = "^3.9" -content-hash = "d9477286cf1bdf683f93d06e49de5cdb4022bbd01cef903fbe861965aab08f59" +python-versions = ">=3.9, <3.11" +content-hash = "50bfa5217ed253ea12cabc67d550ed2f563b19680dc09b07baacccc85173f12b" [metadata.files] anyio = [ {file = "anyio-3.6.1-py3-none-any.whl", hash = "sha256:cb29b9c70620506a9a8f87a309591713446953302d7d995344d0d7c6c0c9a7be"}, {file = "anyio-3.6.1.tar.gz", hash = "sha256:413adf95f93886e442aea925f3ee43baa5a765a64a0f52c6081894f9992fdd0b"}, ] +asn1crypto = [] astroid = [] attrs = [] bandit = [] +beautifulsoup4 = [ + {file = "beautifulsoup4-4.11.1-py3-none-any.whl", hash = "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30"}, + {file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"}, +] black = [] +boto3 = [] +botocore = [] cachetools = [ {file = "cachetools-5.2.0-py3-none-any.whl", hash = "sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db"}, {file = "cachetools-5.2.0.tar.gz", hash = "sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757"}, @@ -1277,33 +1622,28 @@ distlib = [] dparse = [] fastapi = [] filelock = [] +future = [] gitdb = [] gitpython = [] google-api-core = [] -google-api-python-client = [] google-auth = [] -google-auth-httplib2 = [ - {file = "google-auth-httplib2-0.1.0.tar.gz", hash = "sha256:a07c39fd632becacd3f07718dfd6021bf396978f03ad3ce4321d060015cc30ac"}, - {file = "google_auth_httplib2-0.1.0-py2.py3-none-any.whl", hash = "sha256:31e49c36c6b5643b57e82617cb3e021e3e1d2df9da63af67252c02fa9c1f4a10"}, -] -google-auth-oauthlib = [] google-cloud-appengine-logging = [] google-cloud-audit-log = [] +google-cloud-bigquery = [] +google-cloud-bigquery-storage = [] google-cloud-core = [] google-cloud-logging = [] +google-crc32c = [] +google-resumable-media = [] googleapis-common-protos = [] +greenlet = [] grpc-google-iam-v1 = [ {file = "grpc-google-iam-v1-0.12.4.tar.gz", hash = "sha256:3f0ac2c940b9a855d7ce7e31fde28bddb0d9ac362d32d07c67148306931a0e30"}, {file = "grpc_google_iam_v1-0.12.4-py2.py3-none-any.whl", hash = "sha256:312801ae848aeb8408c099ea372b96d253077e7851aae1a9e745df984f81f20c"}, ] grpcio = [] grpcio-status = [] -gspread = [] h11 = [] -httplib2 = [ - {file = "httplib2-0.20.4-py3-none-any.whl", hash = "sha256:8b6a905cb1c79eefd03f8669fd993c36dc341f7c558f056cb5a33b5c2f458543"}, - {file = "httplib2-0.20.4.tar.gz", hash = "sha256:58a98e45b4b1a48273073f905d2961666ecf0fbac4250ea5b47aef259eb5c585"}, -] identify = [] idna = [] iniconfig = [ @@ -1314,6 +1654,7 @@ isort = [ {file = "isort-5.10.1-py3-none-any.whl", hash = "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7"}, {file = "isort-5.10.1.tar.gz", hash = "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951"}, ] +jmespath = [] lazy-object-proxy = [ {file = "lazy-object-proxy-1.7.1.tar.gz", hash = "sha256:d609c75b986def706743cdebe5e47553f4a5a1da9c5ff66d76013ef396b5a8a4"}, {file = "lazy_object_proxy-1.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb8c5fd1684d60a9902c60ebe276da1f2281a318ca16c1d0a96db28f62e9166b"}, @@ -1353,6 +1694,7 @@ lazy-object-proxy = [ {file = "lazy_object_proxy-1.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:677ea950bef409b47e51e733283544ac3d660b709cfce7b187f5ace137960d61"}, {file = "lazy_object_proxy-1.7.1-pp37.pp38-none-any.whl", hash = "sha256:d66906d5785da8e0be7360912e99c9188b70f52c422f9fc18223347235691a84"}, ] +lxml = [] mccabe = [ {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, @@ -1363,7 +1705,8 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] nodeenv = [] -oauthlib = [] +numpy = [] +oscrypto = [] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, @@ -1385,6 +1728,7 @@ py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] +pyarrow = [] pyasn1 = [ {file = "pyasn1-0.4.8-py2.4.egg", hash = "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"}, {file = "pyasn1-0.4.8-py2.5.egg", hash = "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf"}, @@ -1419,6 +1763,7 @@ pycparser = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] +pycryptodomex = [] pydantic = [] pydocstyle = [] pygithub = [] @@ -1426,12 +1771,19 @@ pygments = [] pyjwt = [] pylint = [] pynacl = [] +pyopenssl = [] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] pytest = [] pytest-mock = [] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] +python-dotenv = [] +pytz = [] pyupgrade = [] pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, @@ -1468,13 +1820,15 @@ pyyaml = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +redshift-connector = [] requests = [] -requests-oauthlib = [] rich = [] rsa = [] "ruamel.yaml" = [] "ruamel.yaml.clib" = [] +s3transfer = [] safety = [] +scramp = [] shellingham = [] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, @@ -1486,6 +1840,17 @@ snowballstemmer = [ {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, ] +snowflake-connector-python = [] +snowflake-sqlalchemy = [] +soupsieve = [ + {file = "soupsieve-2.3.2.post1-py3-none-any.whl", hash = "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759"}, + {file = "soupsieve-2.3.2.post1.tar.gz", hash = "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"}, +] +sqlalchemy = [] +sqlalchemy-bigquery = [] +sqlalchemy-redshift = [] +sqlalchemy2-stubs = [] +sqlmodel = [] starlette = [] stevedore = [] tokenize-rt = [] @@ -1503,10 +1868,6 @@ types-pyyaml = [] types-requests = [] types-urllib3 = [] typing-extensions = [] -uritemplate = [ - {file = "uritemplate-4.1.1-py2.py3-none-any.whl", hash = "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e"}, - {file = "uritemplate-4.1.1.tar.gz", hash = "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0"}, -] urllib3 = [] uvicorn = [] virtualenv = [] diff --git a/pyproject.toml b/pyproject.toml index 531d608..905f296 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ # Update me "launch_github_app" = "github_tests_validator_app.bin.github_app_backend:launch_app" [tool.poetry.dependencies] -python = "^3.9" +python = ">=3.9, <3.11" importlib_metadata = {version = ">=1.6.0", python = "<3.8"} typer = {extras = ["all"], version = ">=0.3.2"} rich = ">=10.1.0" @@ -42,15 +42,17 @@ requests = ">=2.22.0" PyGithub = ">=1.55" cryptography = ">=36.0.1" urllib3 = ">=1.26.5" -gspread = "^5.4.0" types-requests = "^2.28.9" pytest-mock = "^3.8.2" types-PyYAML = "^6.0.11" -google-api-python-client = "^2.60.0" -google-auth-httplib2 = "^0.1.0" -google-auth-oauthlib = "^0.5.2" PyYAML = "^6.0" google-cloud-logging = "^3.2.2" +sqlmodel = "^0.0.8" +sqlalchemy-bigquery = "^1.4.4" +redshift-connector = "^2.0.909" +sqlalchemy-redshift = "^0.8.11" +snowflake-sqlalchemy = "^1.4.2" +python-dotenv = "^0.21.0" [tool.poetry.dev-dependencies] darglint = ">=1.8.0" diff --git a/tests/units/test_github_repo_validation.py b/tests/units/test_github_repo_validation.py index a3e32d4..7891b66 100644 --- a/tests/units/test_github_repo_validation.py +++ b/tests/units/test_github_repo_validation.py @@ -1,5 +1,5 @@ import pytest -from github_tests_validator_app.bin.github_repo_validation import get_event, get_student_branch +from github_tests_validator_app.bin.github_repo_validation import get_event, get_user_branch @pytest.mark.parametrize( @@ -25,5 +25,5 @@ def test_get_event(payload, expected): ({"ref": "path"}, "pusher", "path"), ], ) -def test_get_student_branch(payload, trigger, expected): - assert get_student_branch(payload, trigger) == expected +def test_get_user_branch(payload, trigger, expected): + assert get_user_branch(payload, trigger) == expected diff --git a/tests/units/test_utils.py b/tests/units/test_utils.py index fd0d4d1..0c1fb9b 100644 --- a/tests/units/test_utils.py +++ b/tests/units/test_utils.py @@ -2,7 +2,7 @@ import pytest from github import ContentFile -from github_tests_validator_app.lib.models.users import GitHubUser +from github_tests_validator_app.lib.connectors.sqlalchemy_client import User from github_tests_validator_app.lib.utils import get_hash_files, init_github_user_from_github_event @@ -32,12 +32,12 @@ def test_get_hast_files(mocker, contents, expected): "contents,expected", [ ( - {"repository": {"owner": {"login": "test", "id": "1234", "url": "url"}}}, - GitHubUser(LOGIN="test", ID="1234", URL="url"), + {"sender": {"login": "test", "id": "1234", "url": "url"}}, + dict(organization_or_user="test", id="1234", url="url"), ), ( - {"repository": {"owner": {"login": "", "id": "", "url": ""}}}, - GitHubUser(LOGIN="", ID="", URL=""), + {"sender": {"login": "", "id": "", "url": ""}}, + dict(organization_or_user="", id="", url=""), ), ({}, None), ], @@ -45,9 +45,9 @@ def test_get_hast_files(mocker, contents, expected): def test_init_github_user_from_github_event(contents, expected): github_user = init_github_user_from_github_event(contents) assert isinstance(github_user, type(expected)) - if isinstance(github_user, GitHubUser): + if isinstance(github_user, User): assert ( - github_user.LOGIN == expected.LOGIN - and github_user.ID == expected.ID - and github_user.URL == expected.URL + github_user.organization_or_user == expected.organization_or_user + and github_user.id == expected.id + and github_user.url == expected.url )