diff --git a/.tekton/.currency/currency-pipeline.yaml b/.tekton/.currency/currency-pipeline.yaml new file mode 100644 index 00000000..0c4ae0f3 --- /dev/null +++ b/.tekton/.currency/currency-pipeline.yaml @@ -0,0 +1,36 @@ +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: python-currency-pipeline +spec: + params: + - name: revision + type: string + workspaces: + - name: currency-pvc + tasks: + - name: clone-repo + params: + - name: revision + value: $(params.revision) + taskRef: + name: git-clone-task + workspaces: + - name: task-pvc + workspace: currency-pvc + - name: generate-currency-report + runAfter: + - clone-repo + taskRef: + name: generate-currency-report-task + workspaces: + - name: task-pvc + workspace: currency-pvc + - name: upload-currency-report + runAfter: + - generate-currency-report + taskRef: + name: upload-currency-report-task + workspaces: + - name: task-pvc + workspace: currency-pvc diff --git a/.tekton/.currency/currency-pipelinerun.yaml b/.tekton/.currency/currency-pipelinerun.yaml new file mode 100644 index 00000000..151f5403 --- /dev/null +++ b/.tekton/.currency/currency-pipelinerun.yaml @@ -0,0 +1,20 @@ +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: python-currency-pipelinerun +spec: + params: + - name: revision + value: "master" + pipelineRef: + name: python-currency-pipeline + serviceAccountName: currency-serviceaccount + workspaces: + - name: currency-pvc + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi diff --git a/.tekton/.currency/currency-rbac.yaml b/.tekton/.currency/currency-rbac.yaml new file mode 100644 index 00000000..aca210e4 --- /dev/null +++ b/.tekton/.currency/currency-rbac.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: currency-serviceaccount +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: currency-clusterrole +rules: +- apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["get", "list"] +- apiGroups: ["tekton.dev"] + resources: ["taskruns"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: currency-clusterrolebinding +subjects: +- kind: ServiceAccount + name: currency-serviceaccount + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: currency-clusterrole diff --git a/.tekton/.currency/currency-scheduled-eventlistener.yaml b/.tekton/.currency/currency-scheduled-eventlistener.yaml new file mode 100644 index 00000000..a7916e30 --- /dev/null +++ b/.tekton/.currency/currency-scheduled-eventlistener.yaml @@ -0,0 +1,56 @@ +apiVersion: triggers.tekton.dev/v1beta1 +kind: EventListener +metadata: + name: python-currency-cron-listener +spec: + serviceAccountName: tekton-triggers-eventlistener-serviceaccount + triggers: + - name: currency-cron-trigger + template: + ref: python-currency-trigger-template +--- +apiVersion: triggers.tekton.dev/v1beta1 +kind: TriggerTemplate +metadata: + name: python-currency-trigger-template +spec: + resourcetemplates: + - apiVersion: tekton.dev/v1beta1 + kind: PipelineRun + metadata: + generateName: python-currency- + spec: + pipelineRef: + name: python-currency-pipeline + serviceAccountName: currency-serviceaccount + params: + - name: revision + value: "master" + workspaces: + - name: currency-pvc + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: python-currency-cronjob +spec: + schedule: "5 2 * * Mon-Fri" + jobTemplate: + spec: + template: + spec: + containers: + - name: http-request-to-el-svc + # curlimages/curl:8.6.0 + image: curlimages/curl@sha256:f2237028bed58de91f62aea74260bb2a299cf12fbcabc23cfaf125fef276c884 + imagePullPolicy: IfNotPresent + args: ["curl", "-X", "POST", "--data", "{}", "el-python-currency-cron-listener.default.svc.cluster.local:8080"] + restartPolicy: OnFailure +--- diff --git a/.tekton/.currency/currency-tasks.yaml b/.tekton/.currency/currency-tasks.yaml new file mode 100644 index 00000000..6a43f7e1 --- /dev/null +++ b/.tekton/.currency/currency-tasks.yaml @@ -0,0 +1,89 @@ +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: git-clone-task +spec: + params: + - name: revision + type: string + workspaces: + - name: task-pvc + mountPath: /workspace + steps: + - name: clone-repo + # alpine/git:2.43.0 + image: alpine/git@sha256:6ff4de047dcc8f0c7d75d2efff63fbc189e87d2f458305f2cc8f165ff83309cf + script: | + #!/bin/sh + echo "Cloning repo" + cd /workspace && git clone --filter=blob:none --sparse --depth 1 https://github.com/instana/python-sensor -b $(params.revision) + cd python-sensor + git sparse-checkout add .tekton/.currency + ls -lah /workspace/python-sensor +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: generate-currency-report-task +spec: + workspaces: + - name: task-pvc + mountPath: /workspace + steps: + - name: generate-currency-report + # 3.10.13-bookworm + image: python@sha256:c970ff53939772f47b0672e380328afb50d8fd1c0568ed4f82c22effc54244fc + script: | + #!/usr/bin/env bash + cd /workspace/python-sensor/.tekton/.currency + + python -m venv /tmp/venv + source /tmp/venv/bin/activate + pip install -r resources/requirements.txt + + python scripts/generate_report.py + cat docs/report.md +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: upload-currency-report-task +spec: + params: + - name: github-token-secret + default: instanacd-github-api-token + workspaces: + - name: task-pvc + mountPath: /workspace + steps: + - name: upload-currency-report + # alpine/git:2.43.0 + image: alpine/git@sha256:6ff4de047dcc8f0c7d75d2efff63fbc189e87d2f458305f2cc8f165ff83309cf + env: + - name: GH_ENTERPRISE_TOKEN + valueFrom: + secretKeyRef: + name: $(params.github-token-secret) + key: "GH_ENTERPRISE_TOKEN" + script: | + #!/bin/sh + + cd /workspace + git clone https://oauth2:$GH_ENTERPRISE_TOKEN@github.ibm.com/instana/tracer-reports.git + + if [ $? -ne 0 ]; then + echo "The attempt to clone the tracer-reports repository failed, preventing the upload of python tracer currency report." >&2 + exit 1 + fi + + cd tracer-reports + + cp ../python-sensor/.tekton/.currency/docs/report.md ./automated/currency/python/report.md + + git config user.name "Instanacd PAT for GitHub Enterprise" + git config user.email instana.ibm.github.enterprise@ibm.com + + git add . + + git commit -m "chore: Updated python currency report" + git push origin main diff --git a/.tekton/.currency/docs/report.md b/.tekton/.currency/docs/report.md new file mode 100644 index 00000000..7ff39513 --- /dev/null +++ b/.tekton/.currency/docs/report.md @@ -0,0 +1,30 @@ +##### This page is auto-generated. Any change will be overwritten after the next sync. Please apply changes directly to the files in the [python tracer](https://github.com/instana/python-sensor) repo. +## Python supported packages and versions +| Package name | Support Policy | Beta version | Last Supported Version | Latest version | Up-to-date | Cloud Native | +|:---------------------|:-----------------|:---------------|:-------------------------|:-----------------|:-------------|:---------------| +| ASGI | 0-day | No | 3.0 | 3.0 | Yes | No | +| Celery | 30-days | No | 5.4.0 | 5.4.0 | Yes | No | +| Django | 30-days | No | 5.0.6 | 5.0.6 | Yes | No | +| FastAPI | 0-day | No | 0.111.0 | 0.111.0 | Yes | No | +| Flask | 0-day | No | 3.0.3 | 3.0.3 | Yes | No | +| Pyramid | 30-days | No | 2.0.2 | 2.0.2 | Yes | No | +| Sanic | On demand | No | 21.6.2 | 23.12.1 | No | No | +| Starlette | 30-days | No | 0.37.2 | 0.37.2 | Yes | No | +| Tornado | 30-days | No | 5.1.1 | 6.4 | No | No | +| Webapp2 | On demand | No | 2.5.2 | 2.5.2 | Yes | No | +| WSGI | 0-day | No | 1.0.1 | 1.0.1 | Yes | No | +| Aiohttp | 30-days | No | 3.9.5 | 3.9.5 | Yes | No | +| Asynqp | Deprecated | No | 0.6 | 0.6 | Yes | No | +| Boto3 | 0-day | No | 1.34.112 | 1.34.112 | Yes | Yes | +| Google-cloud-pubsub | 30-days | No | 2.1.0 | 2.21.1 | No | Yes | +| Google-cloud-storage | 30-days | No | 2.14.0 | 2.16.0 | No | Yes | +| Grpcio | 30-days | No | 1.64.0 | 1.64.0 | Yes | Yes | +| Mysqlclient | 30-days | No | 2.2.4 | 2.2.4 | Yes | Yes | +| Pika | 30-days | No | 1.3.2 | 1.3.2 | Yes | No | +| PyMySQL | 30-days | No | 1.1.1 | 1.1.1 | Yes | Yes | +| Pymongo | 30-days | No | 4.7.2 | 4.7.2 | Yes | Yes | +| Psycopg2 | 30-days | No | 2.9.9 | 2.9.9 | Yes | No | +| Redis | 30-days | No | 5.0.4 | 5.0.4 | Yes | Yes | +| Requests | 0-day | No | 2.32.2 | 2.32.2 | Yes | Yes | +| SQLAlchemy | 30-days | No | 2.0.30 | 2.0.30 | Yes | Yes | +| Urllib3 | 0-day | No | 2.2.1 | 2.2.1 | Yes | No | \ No newline at end of file diff --git a/.tekton/.currency/resources/requirements.txt b/.tekton/.currency/resources/requirements.txt new file mode 100644 index 00000000..06d8600c --- /dev/null +++ b/.tekton/.currency/resources/requirements.txt @@ -0,0 +1,5 @@ +requests +pandas +beautifulsoup4 +tabulate +kubernetes diff --git a/.tekton/.currency/resources/table.json b/.tekton/.currency/resources/table.json new file mode 100644 index 00000000..05b4f058 --- /dev/null +++ b/.tekton/.currency/resources/table.json @@ -0,0 +1,165 @@ +{ + "table": [ + { + "Package name": "ASGI", + "Support Policy": "0-day", + "Beta version": "No", + "Last Supported Version": "3.0", + "Cloud Native": "No" + }, + { + "Package name": "Celery", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Django", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "FastAPI", + "Support Policy": "0-day", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Flask", + "Support Policy": "0-day", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Pyramid", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Sanic", + "Support Policy": "On demand", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Starlette", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Tornado", + "Support Policy": "30-days", + "Beta version": "No", + "Last Supported Version": "5.1.1", + "Cloud Native": "No" + }, + { + "Package name": "Webapp2", + "Support Policy": "On demand", + "Beta version": "No", + "Last Supported Version": "2.5.2", + "Cloud Native": "No" + }, + { + "Package name": "WSGI", + "Support Policy": "0-day", + "Beta version": "No", + "Last Supported Version": "1.0.1", + "Cloud Native": "No" + }, + { + "Package name": "Aiohttp", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Asynqp", + "Support Policy": "Deprecated", + "Beta version": "No", + "Last Supported Version": "0.6", + "Cloud Native": "No" + }, + { + "Package name": "Boto3", + "Support Policy": "0-day", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Google-cloud-pubsub", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Google-cloud-storage", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Grpcio", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Mysqlclient", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Pika", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "PyMySQL", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Pymongo", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Psycopg2", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Redis", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Requests", + "Support Policy": "0-day", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "SQLAlchemy", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Urllib3", + "Support Policy": "0-day", + "Beta version": "No", + "Cloud Native": "No" + } + ] + } diff --git a/.tekton/.currency/scripts/generate_report.py b/.tekton/.currency/scripts/generate_report.py new file mode 100644 index 00000000..2ae36597 --- /dev/null +++ b/.tekton/.currency/scripts/generate_report.py @@ -0,0 +1,204 @@ +# Standard Libraries +import re +import json + +# Third Party +import requests +import pandas as pd +from bs4 import BeautifulSoup +from kubernetes import client, config + +JSON_FILE = "resources/table.json" +REPORT_FILE = "docs/report.md" +PIP_INDEX_URL = "https://pypi.org/pypi" + +SPEC_MAP = { + "ASGI": "https://asgi.readthedocs.io/en/latest/specs/main.html", + "WSGI": "https://peps.python.org/", +} + + +def get_upstream_version(dependency): + """Get the latest version available upstream""" + if dependency in SPEC_MAP: + # webscrape info from official website + pattern = "(\d+\.\d+\.?\d*)" + + url = SPEC_MAP[dependency] + page = requests.get(url) + soup = BeautifulSoup(page.text, "html.parser") + # ASGI + if "asgi" in url: + text = ( + soup.find(id="version-history") + .findChild("li", string=re.compile(pattern)) + .text + ) + # WSGI + else: + tag = soup.find(id="numerical-index").find_all( + "a", string=re.compile("Web Server Gateway Interface") + )[-1] + text = tag.text + res = re.search(pattern, text) + return res[1] + + else: + # get info using PYPI API + response = requests.get(f"{PIP_INDEX_URL}/{dependency}/json") + response_json = response.json() + latest_version = response_json["info"]["version"] + return latest_version + + +def get_last_supported_version(tekton_ci_output, dependency): + """Get up-to-date supported version""" + pattern = r"-([^\s]+)" + + if dependency == "Psycopg2": + dependency = "psycopg2-binary" + + last_supported_version = re.search( + dependency + pattern, tekton_ci_output, flags=re.I | re.M + ) + + return last_supported_version[1] + + +def isUptodate(last_supported_version, latest_version): + """Check if the supported package is up-to-date""" + if last_supported_version == latest_version: + up_to_date = "Yes" + else: + up_to_date = "No" + + return up_to_date + + +def get_taskruns(namespace, task_name, taskrun_filter): + """Get sorted taskruns filtered based on label_selector""" + group = "tekton.dev" + version = "v1" + plural = "taskruns" + + # access the custom resource from tekton + tektonV1 = client.CustomObjectsApi() + taskruns = tektonV1.list_namespaced_custom_object( + group, + version, + namespace, + plural, + label_selector=f"{group}/task={task_name}, triggers.tekton.dev/trigger=python-tracer-scheduled-pipeline-triggger", + )["items"] + + filtered_taskruns = list(filter(taskrun_filter, taskruns)) + filtered_taskruns.sort( + key=lambda tr: tr["metadata"]["creationTimestamp"], reverse=True + ) + + return filtered_taskruns + + +def process_taskrun_logs( + taskruns, core_v1_client, namespace, task_name, tekton_ci_output +): + """Process taskrun logs""" + for tr in taskruns: + pod_name = tr["status"]["podName"] + taskrun_name = tr["metadata"]["name"] + logs = core_v1_client.read_namespaced_pod_log( + pod_name, namespace, container="step-unittest" + ) + if "Successfully installed" in logs: + print( + f"Retrieving container logs from the successful taskrun pod {pod_name} of taskrun {taskrun_name}.." + ) + if task_name == "python-tracer-unittest-gevent-starlette-task": + match = re.search("Successfully installed .* (starlette-[^\s]+)", logs) + tekton_ci_output += f"{match[1]}\n" + elif task_name == "python-tracer-unittest-default-task": + for line in logs.splitlines(): + if "Successfully installed" in line: + tekton_ci_output += line + break + else: + print( + f"Unable to retrieve container logs from the successful taskrun pod {pod_name} of taskrun {taskrun_name}." + ) + return tekton_ci_output + + +def get_tekton_ci_output(): + """Get the latest successful scheduled tekton pipeline output""" + # config.load_kube_config() + config.load_incluster_config() + + namespace = "default" + core_v1_client = client.CoreV1Api() + + task_name = "python-tracer-unittest-gevent-starlette-task" + taskrun_filter = lambda tr: tr["status"]["conditions"][0]["type"] == "Succeeded" + starlette_taskruns = get_taskruns(namespace, task_name, taskrun_filter) + + tekton_ci_output = process_taskrun_logs( + starlette_taskruns, core_v1_client, namespace, task_name, "" + ) + + task_name = "python-tracer-unittest-default-task" + taskrun_filter = ( + lambda tr: tr["metadata"]["name"].endswith("unittest-default-3") + and tr["status"]["conditions"][0]["type"] == "Succeeded" + ) + default_taskruns = get_taskruns(namespace, task_name, taskrun_filter) + + tekton_ci_output = process_taskrun_logs( + default_taskruns, core_v1_client, namespace, task_name, tekton_ci_output + ) + + return tekton_ci_output + + +def main(): + # Read the JSON file + with open(JSON_FILE) as file: + data = json.load(file) + + items = data["table"] + tekton_ci_output = get_tekton_ci_output() + + for item in items: + package = item["Package name"] + + if "Last Supported Version" not in item: + last_supported_version = get_last_supported_version( + tekton_ci_output, package + ) + item.update({"Last Supported Version": last_supported_version}) + else: + last_supported_version = item["Last Supported Version"] + + latest_version = get_upstream_version(package) + + up_to_date = isUptodate(last_supported_version, latest_version) + + item.update({"Latest version": latest_version, "Up-to-date": up_to_date}) + + # Create a DataFrame from the list of dictionaries + df = pd.DataFrame(items) + df.insert(len(df.columns) - 1, "Cloud Native", df.pop("Cloud Native")) + + # Convert dataframe to markdown + markdown_table = df.to_markdown(index=False) + + disclaimer = f"##### This page is auto-generated. Any change will be overwritten after the next sync. Please apply changes directly to the files in the [python tracer](https://github.com/instana/python-sensor) repo." + title = "## Python supported packages and versions" + + # Combine disclaimer, title, and markdown table with line breaks + final_markdown = disclaimer + "\n" + title + "\n" + markdown_table + + with open(REPORT_FILE, "w") as file: + file.write(final_markdown) + + +if __name__ == "__main__": + main()