Bz/feat/deployment

docker/Dockerfile

@@ -16,7 +16,6 @@ WORKDIR /home/docker_user/workspace
 
 # Install Poetry
 RUN curl -sSL -o install-poetry.py https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py \
-  && echo '8bb6d1f65ed1f19fe500f8baf11328217459a30bd9d94fa5ec3bc9fc0fc345e7 install-poetry.py' | sha256sum --check \
   && POETRY_HOME=/home/docker_user/poetry python install-poetry.py --preview \
   && rm install-poetry.py
 
@@ -25,8 +24,8 @@ ENV PATH="${PATH}:/home/docker_user/.poetry/bin:/home/docker_user/poetry/bin"
 COPY pyproject.toml ./
 COPY poetry.lock ./
 
-RUN poetry install --no-root --no-dev
+RUN poetry install --no-root --only main
 
 COPY . /home/docker_user/workspace/
 
-ENTRYPOINT ["poetry", "run", "github_tests_validator_app"]
+CMD ["poetry", "run", "python3", "server.py"]

docker/cloudbuild.yaml

@@ -0,0 +1,30 @@
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  args:
+  - 'build'
+  - '--tag'
+  - 'europe-west1-docker.pkg.dev/school-of-data-github-app/github-test-validator-app/github_test_validator_app_server:lastest'
+  - "--file=./docker/Dockerfile"
+  - '.'
+  id: Build Artifact
+- name: 'gcr.io/cloud-builders/gcloud'
+  args:
+  - 'run'
+  - 'deploy'
+  - 'github-test-validator-app-server'
+  - '--image'
+  - 'europe-west1-docker.pkg.dev/school-of-data-github-app/github-test-validator-app/github_test_validator_app_server:lastest'
+  - '--region'
+  - 'europe-west1'
+  - '--platform'
+  - 'managed'
+  - '--allow-unauthenticated'
+  - '--service-account=school-of-data-app-dev@school-of-data-github-app.iam.gserviceaccount.com'
+  - '--set-env-vars'
+  - 'GH_APP_ID=230621'
+  - '--set-env-vars'
+  - 'USER_SHARE=bruno.zheng@artefact.com'
+  - '--set-env-vars'
+  - 'ENV=PROD'
+images:
+- 'europe-west1-docker.pkg.dev/school-of-data-github-app/github-test-validator-app/github_test_validator_app_server:lastest'

github_tests_validator_app/bin/github_event_process.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 import logging
 
@@ -16,8 +16,8 @@
     GSHEET_DETAILS_SPREADSHEET,
     USER_SHARE,
 )
-from github_tests_validator_app.lib.connectors.gddrive import GoogleDriveConnector
-from github_tests_validator_app.lib.connectors.gsheet import GSheetConnector
+from github_tests_validator_app.lib.connectors.google_drive import GoogleDriveConnector
+from github_tests_validator_app.lib.connectors.google_sheet import GSheetConnector
 from github_tests_validator_app.lib.models.file import GSheetDetailFile, GSheetFile, WorkSheetFile
 from github_tests_validator_app.lib.models.users import GitHubUser
 from github_tests_validator_app.lib.utils import init_github_user_from_github_event
@@ -48,11 +48,13 @@ def handle_process(payload: Dict[str, Any]) -> str:
 
 
 def init_gsheet_file(
-    google_drive: GoogleDriveConnector, info: Dict[str, Any], parent_id: str, user_share: str
+    google_drive: GoogleDriveConnector,
+    info: Dict[str, Any],
+    parent_id: str,
+    shared_user_list: List[str],
 ) -> GSheetFile:
 
-    gsheet = google_drive.get_gsheet(info["name"], parent_id, user_share)
-
+    gsheet = google_drive.get_gsheet(info["name"], parent_id, shared_user_list)
     list_worksheets = [
         WorkSheetFile(NAME=worksheet["name"], HEADERS=worksheet["headers"])
         for _, worksheet in info["worksheets"].items()
@@ -66,7 +68,7 @@ def init_gsheet_file(
 
 
 def init_gsheet_detail_file(
-    google_drive: GoogleDriveConnector, info: Dict[str, Any], parent_id: str, user_share: str
+    google_drive: GoogleDriveConnector, info: Dict[str, Any], parent_id: str, user_share: List[str]
 ) -> GSheetDetailFile:
 
     gsheet = google_drive.get_gsheet(info["name"], parent_id, user_share)
@@ -94,19 +96,19 @@ def run(payload: Dict[str, Any]) -> Any:
         return
 
     # Init Google Drive connector and folders
-    googe_drive = GoogleDriveConnector()
-    folder_school_of_data = googe_drive.get_gdrive_folder(GDRIVE_MAIN_DIRECTORY_NAME, USER_SHARE)
+    google_drive = GoogleDriveConnector()
+    folder_school_of_data = google_drive.get_gdrive_folder(GDRIVE_MAIN_DIRECTORY_NAME, USER_SHARE)
 
     # Init Google sheets
     gsheet_summary_file = init_gsheet_file(
-        googe_drive, GDRIVE_SUMMARY_SPREADSHEET, folder_school_of_data["id"], USER_SHARE
+        google_drive, GDRIVE_SUMMARY_SPREADSHEET, folder_school_of_data["id"], USER_SHARE
     )
     gsheet_details_file = init_gsheet_detail_file(
-        googe_drive, GSHEET_DETAILS_SPREADSHEET, folder_school_of_data["id"], USER_SHARE
+        google_drive, GSHEET_DETAILS_SPREADSHEET, folder_school_of_data["id"], USER_SHARE
     )
 
     # Init Google sheet connector and worksheets
-    gsheet = GSheetConnector(gsheet_summary_file, gsheet_details_file)
+    gsheet = GSheetConnector(google_drive.credentials, gsheet_summary_file, gsheet_details_file)
 
     # Init GitHubUser
     student_user = init_github_user_from_github_event(payload)

github_tests_validator_app/bin/github_repo_validation.py

@@ -4,14 +4,16 @@
 
 from github import ContentFile
 from github_tests_validator_app.config.config import (
+    GCP_PROJECT_ID,
+    GH_ACCESS_TOKEN_NAME,
     GH_SOLUTION_OWNER,
     GH_SOLUTION_REPO_NAME,
-    GH_SOLUTION_TESTS_ACCESS_TOKEN,
     GH_TESTS_FOLDER_NAME,
     default_message,
 )
 from github_tests_validator_app.lib.connectors.github_connector import GitHubConnector
-from github_tests_validator_app.lib.connectors.gsheet import GSheetConnector
+from github_tests_validator_app.lib.connectors.google_secret_manager import GSecretManager
+from github_tests_validator_app.lib.connectors.google_sheet import GSheetConnector
 from github_tests_validator_app.lib.models.users import GitHubUser
 
 commit_sha_path: Dict[str, List[str]] = {
@@ -91,12 +93,15 @@ def github_repo_validation(
     student_github_connector: GitHubConnector, gsheet: GSheetConnector, payload: Dict[str, Any]
 ) -> None:
 
-    solution_user = GitHubUser(LOGIN=str(GH_SOLUTION_OWNER))
+    solution_user = GitHubUser(LOGIN=GH_SOLUTION_OWNER)
+    gsecret_manager = GSecretManager(GCP_PROJECT_ID)
+    acces_token = gsecret_manager.get_access_secret_version(GH_ACCESS_TOKEN_NAME)
+
     solution_github_connector = GitHubConnector(
         user=solution_user,
         repo_name=GH_SOLUTION_REPO_NAME,
         branch_name="main",
-        access_token=GH_SOLUTION_TESTS_ACCESS_TOKEN,
+        access_token=acces_token,
     )
     if not solution_github_connector:
         gsheet.add_new_repo_valid_result(

github_tests_validator_app/bin/student_challenge_results_validation.py

@@ -4,7 +4,7 @@
 
 from github_tests_validator_app.config.config import CHALLENGE_DIR
 from github_tests_validator_app.lib.connectors.github_connector import GitHubConnector
-from github_tests_validator_app.lib.connectors.gsheet import GSheetConnector
+from github_tests_validator_app.lib.connectors.google_sheet import GSheetConnector
 from github_tests_validator_app.lib.models.pytest_result import PytestResult
 
 

github_tests_validator_app/config/__init__.py

@@ -1,12 +1,20 @@
 import logging
+import os
 
-FORMAT = "%(asctime)s - %(levelname)s: %(message)s"
-DATEFMT = "%H:%M:%S"
-logging.basicConfig(
-    format=FORMAT,
-    level=logging.INFO,
-    datefmt=DATEFMT,
-)
+import google.cloud.logging
 
-if logging.getLogger("uvicorn") and logging.getLogger("uvicorn").handlers:
-    logging.getLogger("uvicorn").removeHandler(logging.getLogger("uvicorn").handlers[0])
+if os.getenv("ENV", "") == "PROD":
+    logging_client = google.cloud.logging.Client()
+    logging_client.get_default_handler()
+    logging_client.setup_logging()
+else:
+    FORMAT = "%(asctime)s - %(levelname)s: %(message)s"
+    DATEFMT = "%H:%M:%S"
+    logging.basicConfig(
+        format=FORMAT,
+        level=logging.INFO,
+        datefmt=DATEFMT,
+    )
+
+    if logging.getLogger("uvicorn") and logging.getLogger("uvicorn").handlers:
+        logging.getLogger("uvicorn").removeHandler(logging.getLogger("uvicorn").handlers[0])

github_tests_validator_app/config/config.py

@@ -6,22 +6,25 @@
 
 # GitHub
 GH_APP_ID = cast(str, os.getenv("GH_APP_ID", None))
-GH_APP_KEY_PATH = os.getenv("GH_APP_KEY", "")
-GH_SOLUTION_TESTS_ACCESS_TOKEN = cast(str, os.getenv("SOLUTION_TESTS_ACCESS_TOKEN", None))
 GH_SOLUTION_OWNER = "artefactory-fr"
 GH_SOLUTION_REPO_NAME = "school_of_data_tests"
 GH_TESTS_FOLDER_NAME = "tests"
 GH_API = "https://api.github.com/repos"
 GH_ALL_ARTIFACT_ENDPOINT = "actions/artifacts"
 
+# GCP
+GCP_PROJECT_ID = "school-of-data-github-app"
+
+# Google Secrets
+GH_APP_KEY_NAME = "github_test_validator_app_key"
+GH_ACCESS_TOKEN_NAME = "github_personal_access_token"
+
 # Google Drive
-GDRIVE_CREDENTIALS_PATH = "credentials.json"
 GDRIVE_MAIN_DIRECTORY_NAME = "school_of_data_results"
 
 # Google Sheet
-GSHEET_SA_JSON = cast(str, os.getenv("GSHEET_SA_JSON", None))
-GDRIVE_HIERARCHY_APTH = "github_tests_validator_app/config/data/gdrive_hierarchy.yml"
-with open(GDRIVE_HIERARCHY_APTH) as file:
+GDRIVE_HIERARCHY_PATH = "github_tests_validator_app/config/data/gdrive_hierarchy.yml"
+with open(GDRIVE_HIERARCHY_PATH) as file:
     data = yaml.safe_load(file)
 
 GDRIVE_SUMMARY_SPREADSHEET = data["gdrive_summary_spreadsheet"]
@@ -38,4 +41,4 @@
 # Common
 CHALLENGE_DIR = "tests/tests/"
 DATE_FORMAT = "%d/%m/%Y %H:%M:%S"
-USER_SHARE = os.getenv("USER_SHARE", "")
+USER_SHARE = os.getenv("USER_SHARE", "").split(",")

github_tests_validator_app/lib/connectors/github_connector.py

@@ -8,11 +8,13 @@
 import requests
 from github import ContentFile, Github, GithubIntegration, Repository
 from github_tests_validator_app.config.config import (
+    GCP_PROJECT_ID,
     GH_ALL_ARTIFACT_ENDPOINT,
     GH_API,
     GH_APP_ID,
-    GH_APP_KEY_PATH,
+    GH_APP_KEY_NAME,
 )
+from github_tests_validator_app.lib.connectors.google_secret_manager import GSecretManager
 from github_tests_validator_app.lib.models.users import GitHubUser
 from github_tests_validator_app.lib.utils import get_hash_files
 
@@ -40,12 +42,13 @@ def __init__(
 
     def set_git_integration(self) -> None:
 
-        with open(GH_APP_KEY_PATH) as f:
-            GH_APP_KEY = f.read()
-            self.git_integration = GithubIntegration(
-                GH_APP_ID,
-                GH_APP_KEY,
-            )
+        gsecret_manager = GSecretManager(GCP_PROJECT_ID)
+        github_app_key = gsecret_manager.get_access_secret_version(GH_APP_KEY_NAME)
+
+        self.git_integration = GithubIntegration(
+            GH_APP_ID,
+            github_app_key,
+        )
 
     def set_access_token(self, repo_name: str) -> None:
         self.ACCESS_TOKEN = self.git_integration.get_access_token(