From 19892bb643689be694a293bebd83e29d51624277 Mon Sep 17 00:00:00 2001 From: Varsha GS Date: Wed, 8 May 2024 13:43:34 +0530 Subject: [PATCH 1/9] currency: automate python tracer currency report generation Signed-off-by: Varsha GS --- .tekton/.currency/currency-pipeline.yaml | 36 ++++ .tekton/.currency/currency-pipelinerun.yaml | 20 +++ .tekton/.currency/currency-rbac.yaml | 29 +++ .tekton/.currency/currency-tasks.yaml | 89 ++++++++++ .tekton/.currency/docs/report.md | 28 +++ .tekton/.currency/scripts/generate_report.py | 138 +++++++++++++++ .../.currency/scripts/get-tekton-ci-output.sh | 16 ++ .tekton/.currency/utils/requirements.txt | 4 + .tekton/.currency/utils/table.json | 165 ++++++++++++++++++ .tekton/.currency/utils/tekton-ci-output.txt | 3 + 10 files changed, 528 insertions(+) create mode 100644 .tekton/.currency/currency-pipeline.yaml create mode 100644 .tekton/.currency/currency-pipelinerun.yaml create mode 100644 .tekton/.currency/currency-rbac.yaml create mode 100644 .tekton/.currency/currency-tasks.yaml create mode 100644 .tekton/.currency/docs/report.md create mode 100644 .tekton/.currency/scripts/generate_report.py create mode 100644 .tekton/.currency/scripts/get-tekton-ci-output.sh create mode 100644 .tekton/.currency/utils/requirements.txt create mode 100644 .tekton/.currency/utils/table.json create mode 100644 .tekton/.currency/utils/tekton-ci-output.txt diff --git a/.tekton/.currency/currency-pipeline.yaml b/.tekton/.currency/currency-pipeline.yaml new file mode 100644 index 00000000..0c4ae0f3 --- /dev/null +++ b/.tekton/.currency/currency-pipeline.yaml @@ -0,0 +1,36 @@ +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: python-currency-pipeline +spec: + params: + - name: revision + type: string + workspaces: + - name: currency-pvc + tasks: + - name: clone-repo + params: + - name: revision + value: $(params.revision) + taskRef: + name: git-clone-task + workspaces: + - name: task-pvc + workspace: currency-pvc + - name: generate-currency-report + runAfter: + - clone-repo + taskRef: + name: generate-currency-report-task + workspaces: + - name: task-pvc + workspace: currency-pvc + - name: upload-currency-report + runAfter: + - generate-currency-report + taskRef: + name: upload-currency-report-task + workspaces: + - name: task-pvc + workspace: currency-pvc diff --git a/.tekton/.currency/currency-pipelinerun.yaml b/.tekton/.currency/currency-pipelinerun.yaml new file mode 100644 index 00000000..a0c2e162 --- /dev/null +++ b/.tekton/.currency/currency-pipelinerun.yaml @@ -0,0 +1,20 @@ +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: python-currency-pipelinerun +spec: + params: + - name: revision + value: "currency-update" + pipelineRef: + name: python-currency-pipeline + serviceAccountName: currency-serviceaccount + workspaces: + - name: currency-pvc + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi diff --git a/.tekton/.currency/currency-rbac.yaml b/.tekton/.currency/currency-rbac.yaml new file mode 100644 index 00000000..b0b32765 --- /dev/null +++ b/.tekton/.currency/currency-rbac.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: currency-serviceaccount +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: currency-clusterrole +rules: +- apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["get", "list", "watch"] +- apiGroups: ["tekton.dev"] + resources: ["taskruns"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: currency-clusterrolebinding +subjects: +- kind: ServiceAccount + name: currency-serviceaccount + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: currency-clusterrole diff --git a/.tekton/.currency/currency-tasks.yaml b/.tekton/.currency/currency-tasks.yaml new file mode 100644 index 00000000..5b345a15 --- /dev/null +++ b/.tekton/.currency/currency-tasks.yaml @@ -0,0 +1,89 @@ +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: git-clone-task +spec: + params: + - name: revision + type: string + workspaces: + - name: task-pvc + mountPath: /workspace + steps: + - name: clone-repo + # alpine/git:2.43.0 + image: alpine/git@sha256:6ff4de047dcc8f0c7d75d2efff63fbc189e87d2f458305f2cc8f165ff83309cf + script: | + #!/bin/sh + echo "Cloning repo" + cd /workspace && git clone --filter=blob:none --sparse --depth 1 https://github.com/instana/python-sensor -b $(params.revision) + cd python-sensor + git sparse-checkout add .tekton/.currency + ls -lah /workspace/python-sensor +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: generate-currency-report-task +spec: + workspaces: + - name: task-pvc + mountPath: /workspace + steps: + - name: generate-currency-report + # 3.10.13-bookworm + image: python@sha256:c970ff53939772f47b0672e380328afb50d8fd1c0568ed4f82c22effc54244fc + script: | + #!/bin/bash + /usr/bin/curl -LO https://storage.googleapis.com/kubernetes-release/release/$(/usr/bin/curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ + chmod +x ./kubectl && \ + mv ./kubectl /usr/local/bin/kubectl + kubectl version + + cd /workspace/python-sensor/.tekton/.currency + + python -m venv /tmp/venv + source /tmp/venv/bin/activate + pip install -r utils/requirements.txt + + python scripts/generate_report.py + echo "Generated report..." +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: upload-currency-report-task +spec: + params: + - name: github-token-secret + default: instanacd-github-api-token + workspaces: + - name: task-pvc + mountPath: /workspace + steps: + - name: upload-currency-report + # alpine/git:2.43.0 + image: alpine/git@sha256:6ff4de047dcc8f0c7d75d2efff63fbc189e87d2f458305f2cc8f165ff83309cf + env: + - name: GH_ENTERPRISE_TOKEN + valueFrom: + secretKeyRef: + name: $(params.github-token-secret) + key: "GH_ENTERPRISE_TOKEN" + script: | + #!/bin/sh + + cd /workspace + git clone https://oauth2:$GH_ENTERPRISE_TOKEN@github.ibm.com/instana/tracer-reports.git tracer-reports + cd tracer-reports + git pull origin main + + cp ../python-sensor/.tekton/.currency/docs/report.md ./automated/currency/python/report.md + + git config user.name "Instanacd PAT for GitHub Enterprise" + git config user.email instana.ibm.github.enterprise@ibm.com + + git add . + + git commit -m "Updated python currency report" + git push origin main diff --git a/.tekton/.currency/docs/report.md b/.tekton/.currency/docs/report.md new file mode 100644 index 00000000..f7507415 --- /dev/null +++ b/.tekton/.currency/docs/report.md @@ -0,0 +1,28 @@ +| Package name | Support Policy | Beta version | Last Supported Version | Latest version | Up-to-date | Cloud Native | +|:---------------------|:-----------------|:---------------|:-------------------------|:-----------------|:-------------|:---------------| +| ASGI | 0-day | No | 3.0 | 3.0 | Yes | No | +| Celery | 30-days | No | 5.4.0 | 5.4.0 | Yes | No | +| Django | 30-days | No | 5.0.4 | 5.0.4 | Yes | No | +| FastAPI | 0-day | No | 0.110.2 | 0.110.2 | Yes | No | +| Flask | 0-day | No | 3.0.3 | 3.0.3 | Yes | No | +| Pyramid | 30-days | No | 2.0.2 | 2.0.2 | Yes | No | +| Sanic | On demand | No | 21.6.2 | 23.12.1 | No | No | +| Starlette | 30-days | No | 0.37.2 | 0.37.2 | Yes | No | +| Tornado | 30-days | No | 5.1.1 | 6.4 | No | No | +| Webapp2 | On demand | No | 2.5.2 | 2.5.2 | Yes | No | +| WSGI | 0-day | No | 1.0.1 | 1.0.1 | Yes | No | +| Aiohttp | 30-days | No | 3.9.5 | 3.9.5 | Yes | No | +| Asynqp | Deprecated | No | 0.6 | 0.6 | Yes | No | +| Boto3 | 0-day | No | 1.34.88 | 1.34.88 | Yes | Yes | +| Google-cloud-pubsub | 30-days | No | 2.1.0 | 2.21.1 | No | Yes | +| Google-cloud-storage | 30-days | No | 2.14.0 | 2.16.0 | No | Yes | +| Grpcio | 30-days | No | 1.62.2 | 1.62.2 | Yes | Yes | +| Mysqlclient | 30-days | No | 2.2.4 | 2.2.4 | Yes | Yes | +| Pika | 30-days | No | 1.3.2 | 1.3.2 | Yes | No | +| PyMySQL | 30-days | No | 1.1.0 | 1.1.0 | Yes | Yes | +| Pymongo | 30-days | No | 4.6.3 | 4.6.3 | Yes | Yes | +| Psycopg2 | 30-days | No | 2.9.9 | 2.9.9 | Yes | No | +| Redis | 30-days | No | 5.0.3 | 5.0.3 | Yes | Yes | +| Requests | 0-day | No | 2.31.0 | 2.31.0 | Yes | Yes | +| SQLAlchemy | 30-days | No | 2.0.29 | 2.0.29 | Yes | Yes | +| Urllib3 | 0-day | No | 2.2.1 | 2.2.1 | Yes | No | \ No newline at end of file diff --git a/.tekton/.currency/scripts/generate_report.py b/.tekton/.currency/scripts/generate_report.py new file mode 100644 index 00000000..a0a2597c --- /dev/null +++ b/.tekton/.currency/scripts/generate_report.py @@ -0,0 +1,138 @@ +# Standard Libraries +import re +import json +from os import system +from datetime import date + +# Third Party +import requests +import pandas as pd +from bs4 import BeautifulSoup + + +JSON_FILE = "utils/table.json" +REPORT_FILE = "docs/report.md" +TEKTON_CI_OUT_FILE = "utils/tekton-ci-output.txt" +TEKTON_CI_OUT_SCRIPT = "scripts/get-tekton-ci-output.sh" +PIP_INDEX_URL = "https://pypi.org/pypi" + +SPEC_MAP = { + "ASGI": "https://asgi.readthedocs.io/en/latest/specs/main.html", + "WSGI": "https://peps.python.org/", +} + + +def get_upstream_version(dependency): + """get the latest version available upstream""" + if dependency in SPEC_MAP: + # webscrape info from official website + pattern = "(\d+\.\d+\.?\d*)" + + url = SPEC_MAP[dependency] + page = requests.get(url) + soup = BeautifulSoup(page.text, "html.parser") + # ASGI + if "asgi" in url: + text = ( + soup.find(id="version-history") + .findChild("li", string=re.compile(pattern)) + .text + ) + # WSGI + else: + tag = soup.find(id="numerical-index").find_all( + "a", string=re.compile("Web Server Gateway Interface") + )[-1] + text = tag.text + res = re.search(pattern, text) + return res[1] + + else: + # get info using PYPI API + response = requests.get(f"{PIP_INDEX_URL}/{dependency}/json") + response_json = response.json() + latest_version = response_json["info"]["version"] + return latest_version + + +## Get the tekton ci output of the installed python dependencies +system("bash " + TEKTON_CI_OUT_SCRIPT) + +with open(TEKTON_CI_OUT_FILE) as file: + content = file.read() + + +def get_last_supported_version(dependency): + """get up-to-date supported version""" + pattern = r"-([^\s]+)" + + if dependency == "Psycopg2": + dependency = "psycopg2-binary" + + last_supported_version = re.search(dependency + pattern, content, flags=re.I | re.M) + + return last_supported_version[1] + + +def isUptodate(last_supported_version, latest_version): + if last_supported_version == latest_version: + up_to_date = "Yes" + else: + up_to_date = "No" + + return up_to_date + + +# Read the JSON file +with open(JSON_FILE) as file: + data = json.load(file) + + +items = data["table"] + +for index in range(len(items)): + item = items[index] + package = item["package_name"] + + if "last_supported_version" not in item: + last_supported_version = get_last_supported_version(package) + item.update({"last_supported_version": last_supported_version}) + else: + last_supported_version = item["last_supported_version"] + + latest_version = get_upstream_version(package) + item.update({"latest_version": latest_version}) + + up_to_date = isUptodate(last_supported_version, latest_version) + + item.update({"up_to_date": up_to_date}) + + +# Create a DataFrame from the list of dictionaries +df = pd.DataFrame(items) +df.insert(len(df.columns) - 1, "cloud_native", df.pop("cloud_native")) + +# Rename Columns +df.columns = [ + "Package name", + "Support Policy", + "Beta version", + "Last Supported Version", + "Latest version", + "Up-to-date", + "Cloud Native", +] + +# Convert dataframe to markdown +markdown_table = df.to_markdown(index=False) + +current_date = date.today().strftime("%b %d, %Y") + +disclaimer = f"This page is auto-generated. Any change will be overwritten after the next sync. Please apply changes directly to the files in the [python tracer](https://github.com/instana/python-sensor) repo. Last updated on **{current_date}**." +title = "## Python supported packages and versions" + +# Combine disclaimer, title, and markdown table with line breaks +final_markdown = disclaimer + "\n" + title + "\n" + markdown_table + +with open(REPORT_FILE, "w") as file: + file.write(final_markdown) diff --git a/.tekton/.currency/scripts/get-tekton-ci-output.sh b/.tekton/.currency/scripts/get-tekton-ci-output.sh new file mode 100644 index 00000000..4be03a11 --- /dev/null +++ b/.tekton/.currency/scripts/get-tekton-ci-output.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +TEKTON_CI_OUT_FILE=utils/tekton-ci-output.txt + +successful_taskruns=( $(kubectl get taskrun --sort-by=.metadata.creationTimestamp | grep "^python-trace\w*-unittest-default-3" | grep -v "pr\|Failed" | awk '{print $1}') ) + +for ((i=${#successful_taskruns[@]}-1; i>=0; i--)); do + pod_name=$(kubectl get taskrun "${successful_taskruns[$i]}" -o jsonpath='{.status.podName}') + ci_output=$(kubectl logs ${pod_name} -c step-unittest | grep "Successfully installed") + if [ -n "${ci_output}" ]; then + latest_successful_taskrun_pod=$pod_name + break + fi +done + +kubectl logs ${latest_successful_taskrun_pod} -c step-unittest | grep "Successfully installed" > ${TEKTON_CI_OUT_FILE} diff --git a/.tekton/.currency/utils/requirements.txt b/.tekton/.currency/utils/requirements.txt new file mode 100644 index 00000000..9c4283bf --- /dev/null +++ b/.tekton/.currency/utils/requirements.txt @@ -0,0 +1,4 @@ +requests +pandas +beautifulsoup4 +tabulate diff --git a/.tekton/.currency/utils/table.json b/.tekton/.currency/utils/table.json new file mode 100644 index 00000000..8e6b773c --- /dev/null +++ b/.tekton/.currency/utils/table.json @@ -0,0 +1,165 @@ +{ + "table": [ + { + "package_name": "ASGI", + "support_policy": "0-day", + "beta_version": "No", + "last_supported_version": "3.0", + "cloud_native": "No" + }, + { + "package_name": "Celery", + "support_policy": "30-days", + "beta_version": "No", + "cloud_native": "No" + }, + { + "package_name": "Django", + "support_policy": "30-days", + "beta_version": "No", + "cloud_native": "No" + }, + { + "package_name": "FastAPI", + "support_policy": "0-day", + "beta_version": "No", + "cloud_native": "No" + }, + { + "package_name": "Flask", + "support_policy": "0-day", + "beta_version": "No", + "cloud_native": "No" + }, + { + "package_name": "Pyramid", + "support_policy": "30-days", + "beta_version": "No", + "cloud_native": "No" + }, + { + "package_name": "Sanic", + "support_policy": "On demand", + "beta_version": "No", + "cloud_native": "No" + }, + { + "package_name": "Starlette", + "support_policy": "30-days", + "beta_version": "No", + "cloud_native": "No" + }, + { + "package_name": "Tornado", + "support_policy": "30-days", + "beta_version": "No", + "last_supported_version": "5.1.1", + "cloud_native": "No" + }, + { + "package_name": "Webapp2", + "support_policy": "On demand", + "beta_version": "No", + "last_supported_version": "2.5.2", + "cloud_native": "No" + }, + { + "package_name": "WSGI", + "support_policy": "0-day", + "beta_version": "No", + "last_supported_version": "1.0.1", + "cloud_native": "No" + }, + { + "package_name": "Aiohttp", + "support_policy": "30-days", + "beta_version": "No", + "cloud_native": "No" + }, + { + "package_name": "Asynqp", + "support_policy": "Deprecated", + "beta_version": "No", + "last_supported_version": "0.6", + "cloud_native": "No" + }, + { + "package_name": "Boto3", + "support_policy": "0-day", + "beta_version": "No", + "cloud_native": "Yes" + }, + { + "package_name": "Google-cloud-pubsub", + "support_policy": "30-days", + "beta_version": "No", + "cloud_native": "Yes" + }, + { + "package_name": "Google-cloud-storage", + "support_policy": "30-days", + "beta_version": "No", + "cloud_native": "Yes" + }, + { + "package_name": "Grpcio", + "support_policy": "30-days", + "beta_version": "No", + "cloud_native": "Yes" + }, + { + "package_name": "Mysqlclient", + "support_policy": "30-days", + "beta_version": "No", + "cloud_native": "Yes" + }, + { + "package_name": "Pika", + "support_policy": "30-days", + "beta_version": "No", + "cloud_native": "No" + }, + { + "package_name": "PyMySQL", + "support_policy": "30-days", + "beta_version": "No", + "cloud_native": "Yes" + }, + { + "package_name": "Pymongo", + "support_policy": "30-days", + "beta_version": "No", + "cloud_native": "Yes" + }, + { + "package_name": "Psycopg2", + "support_policy": "30-days", + "beta_version": "No", + "cloud_native": "No" + }, + { + "package_name": "Redis", + "support_policy": "30-days", + "beta_version": "No", + "cloud_native": "Yes" + }, + { + "package_name": "Requests", + "support_policy": "0-day", + "beta_version": "No", + "cloud_native": "Yes" + }, + { + "package_name": "SQLAlchemy", + "support_policy": "30-days", + "beta_version": "No", + "cloud_native": "Yes" + }, + { + "package_name": "Urllib3", + "support_policy": "0-day", + "beta_version": "No", + "cloud_native": "No" + } + ] + } diff --git a/.tekton/.currency/utils/tekton-ci-output.txt b/.tekton/.currency/utils/tekton-ci-output.txt new file mode 100644 index 00000000..025db350 --- /dev/null +++ b/.tekton/.currency/utils/tekton-ci-output.txt @@ -0,0 +1,3 @@ +[unittest] Successfully installed pip-24.0 +[unittest] Successfully installed autowrapt-1.0 basictracer-3.2.0 certifi-2024.2.2 charset-normalizer-3.3.2 fysom-2.1.6 idna-3.7 instana-2.3.0 opentracing-2.4.0 protobuf-4.25.3 requests-2.31.0 six-1.16.0 urllib3-2.2.1 wrapt-1.16.0 +[unittest] Successfully installed Django-5.0.4 Jinja2-3.1.3 PasteDeploy-3.1.0 PyMySQL-1.1.0 Werkzeug-3.0.2 aiofiles-23.2.1 aiohttp-3.9.5 aiosignal-1.3.1 amqp-5.2.0 annotated-types-0.6.0 anyio-4.3.0 asgiref-3.8.1 async-timeout-4.0.3 attrs-23.2.0 billiard-4.2.0 blinker-1.7.0 boto3-1.34.88 botocore-1.34.88 cachetools-5.3.3 celery-5.4.0 cffi-1.16.0 click-8.1.7 click-didyoumean-0.3.1 click-plugins-1.1.1 click-repl-0.3.0 coverage-7.4.4 cryptography-42.0.5 dnspython-2.6.1 exceptiongroup-1.2.1 fastapi-0.110.2 flask-3.0.3 frozenlist-1.4.1 google-api-core-1.34.1 google-auth-2.29.0 google-cloud-core-2.4.1 google-cloud-pubsub-2.1.0 google-cloud-storage-2.14.0 google-crc32c-1.5.0 google-resumable-media-2.7.0 googleapis-common-protos-1.63.0 greenlet-3.0.3 grpc-google-iam-v1-0.12.7 grpcio-1.62.2 grpcio-status-1.48.2 h11-0.14.0 httptools-0.6.1 hupper-1.12.1 iniconfig-2.0.0 itsdangerous-2.2.0 jmespath-1.0.1 kombu-5.3.7 libcst-1.3.1 lxml-5.2.1 markupsafe-2.1.5 mock-5.1.0 moto-5.0.5 multidict-5.2.0 mysqlclient-2.2.4 packaging-24.0 pika-1.3.2 plaster-1.1.2 plaster-pastedeploy-1.0.1 pluggy-1.5.0 prompt-toolkit-3.0.43 proto-plus-1.23.0 protobuf-3.20.3 psycopg2-binary-2.9.9 pyasn1-0.6.0 pyasn1-modules-0.4.0 pycparser-2.22 pydantic-2.7.0 pydantic-core-2.18.1 pymongo-4.6.3 pyramid-2.0.2 pytest-8.1.1 python-dateutil-2.9.0.post0 pytz-2024.1 pyyaml-6.0.1 redis-5.0.3 requests-mock-1.12.1 responses-0.17.0 rsa-4.9 s3transfer-0.10.1 sanic-21.6.2 sanic-routing-0.7.2 sniffio-1.3.1 spyne-2.14.0 sqlalchemy-2.0.29 sqlparse-0.5.0 starlette-0.37.2 tomli-2.0.1 translationstring-1.4 typing-extensions-4.11.0 tzdata-2024.1 ujson-5.9.0 uvicorn-0.29.0 uvloop-0.19.0 venusian-3.1.0 vine-5.1.0 wcwidth-0.2.13 webob-1.8.7 websockets-12.0 xmltodict-0.13.0 yarl-1.9.4 zope.deprecation-5.0 zope.interface-6.3 From ea9aa41e6537c83958e59b39c7a52b5d59688f77 Mon Sep 17 00:00:00 2001 From: Varsha GS Date: Wed, 8 May 2024 13:44:03 +0530 Subject: [PATCH 2/9] currency: add scheduled event listener Signed-off-by: Varsha GS --- .../currency-scheduled-eventlistener.yaml | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .tekton/.currency/currency-scheduled-eventlistener.yaml diff --git a/.tekton/.currency/currency-scheduled-eventlistener.yaml b/.tekton/.currency/currency-scheduled-eventlistener.yaml new file mode 100644 index 00000000..50d49c95 --- /dev/null +++ b/.tekton/.currency/currency-scheduled-eventlistener.yaml @@ -0,0 +1,56 @@ +apiVersion: triggers.tekton.dev/v1beta1 +kind: EventListener +metadata: + name: python-currency-cron-listener +spec: + serviceAccountName: tekton-triggers-eventlistener-serviceaccount + triggers: + - name: currency-cron-trigger + template: + ref: python-currency-trigger-template +--- +apiVersion: triggers.tekton.dev/v1beta1 +kind: TriggerTemplate +metadata: + name: python-currency-trigger-template +spec: + resourcetemplates: + - apiVersion: tekton.dev/v1beta1 + kind: PipelineRun + metadata: + generateName: python-currency- + spec: + pipelineRef: + name: python-currency-pipeline + serviceAccountName: currency-serviceaccount + params: + - name: revision + value: "currency-update" + workspaces: + - name: currency-pvc + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: python-currency-cronjob +spec: + schedule: "5 2 * * Mon-Fri" + jobTemplate: + spec: + template: + spec: + containers: + - name: http-request-to-el-svc + # curlimages/curl:8.6.0 + image: curlimages/curl@sha256:f2237028bed58de91f62aea74260bb2a299cf12fbcabc23cfaf125fef276c884 + imagePullPolicy: IfNotPresent + args: ["curl", "-X", "POST", "--data", "{}", "el-python-currency-cron-listener.default.svc.cluster.local:8080"] + restartPolicy: OnFailure +--- From 73657920ebf0769cef3aaf0ca3cc70a6af44e3d4 Mon Sep 17 00:00:00 2001 From: Varsha GS Date: Sun, 12 May 2024 21:36:33 +0530 Subject: [PATCH 3/9] chore(currency): minor fixes Signed-off-by: Varsha GS --- .tekton/.currency/currency-tasks.yaml | 11 +- .../{utils => resources}/requirements.txt | 1 + .tekton/.currency/resources/table.json | 165 ++++++++++++++++++ .../{utils => resources}/tekton-ci-output.txt | 0 .tekton/.currency/scripts/generate_report.py | 2 +- .../.currency/scripts/get-tekton-ci-output.sh | 2 +- .tekton/.currency/utils/table.json | 165 ------------------ 7 files changed, 173 insertions(+), 173 deletions(-) rename .tekton/.currency/{utils => resources}/requirements.txt (78%) create mode 100644 .tekton/.currency/resources/table.json rename .tekton/.currency/{utils => resources}/tekton-ci-output.txt (100%) delete mode 100644 .tekton/.currency/utils/table.json diff --git a/.tekton/.currency/currency-tasks.yaml b/.tekton/.currency/currency-tasks.yaml index 5b345a15..458b03fa 100644 --- a/.tekton/.currency/currency-tasks.yaml +++ b/.tekton/.currency/currency-tasks.yaml @@ -34,7 +34,7 @@ spec: # 3.10.13-bookworm image: python@sha256:c970ff53939772f47b0672e380328afb50d8fd1c0568ed4f82c22effc54244fc script: | - #!/bin/bash + #!/usr/bin/env bash /usr/bin/curl -LO https://storage.googleapis.com/kubernetes-release/release/$(/usr/bin/curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ chmod +x ./kubectl && \ mv ./kubectl /usr/local/bin/kubectl @@ -44,10 +44,10 @@ spec: python -m venv /tmp/venv source /tmp/venv/bin/activate - pip install -r utils/requirements.txt + pip install -r resources/requirements.txt python scripts/generate_report.py - echo "Generated report..." + cat docs/report.md --- apiVersion: tekton.dev/v1beta1 kind: Task @@ -74,9 +74,8 @@ spec: #!/bin/sh cd /workspace - git clone https://oauth2:$GH_ENTERPRISE_TOKEN@github.ibm.com/instana/tracer-reports.git tracer-reports + git clone https://oauth2:$GH_ENTERPRISE_TOKEN@github.ibm.com/instana/tracer-reports.git cd tracer-reports - git pull origin main cp ../python-sensor/.tekton/.currency/docs/report.md ./automated/currency/python/report.md @@ -85,5 +84,5 @@ spec: git add . - git commit -m "Updated python currency report" + git commit -m "chore: Updated python currency report" git push origin main diff --git a/.tekton/.currency/utils/requirements.txt b/.tekton/.currency/resources/requirements.txt similarity index 78% rename from .tekton/.currency/utils/requirements.txt rename to .tekton/.currency/resources/requirements.txt index 9c4283bf..06d8600c 100644 --- a/.tekton/.currency/utils/requirements.txt +++ b/.tekton/.currency/resources/requirements.txt @@ -2,3 +2,4 @@ requests pandas beautifulsoup4 tabulate +kubernetes diff --git a/.tekton/.currency/resources/table.json b/.tekton/.currency/resources/table.json new file mode 100644 index 00000000..05b4f058 --- /dev/null +++ b/.tekton/.currency/resources/table.json @@ -0,0 +1,165 @@ +{ + "table": [ + { + "Package name": "ASGI", + "Support Policy": "0-day", + "Beta version": "No", + "Last Supported Version": "3.0", + "Cloud Native": "No" + }, + { + "Package name": "Celery", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Django", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "FastAPI", + "Support Policy": "0-day", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Flask", + "Support Policy": "0-day", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Pyramid", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Sanic", + "Support Policy": "On demand", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Starlette", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Tornado", + "Support Policy": "30-days", + "Beta version": "No", + "Last Supported Version": "5.1.1", + "Cloud Native": "No" + }, + { + "Package name": "Webapp2", + "Support Policy": "On demand", + "Beta version": "No", + "Last Supported Version": "2.5.2", + "Cloud Native": "No" + }, + { + "Package name": "WSGI", + "Support Policy": "0-day", + "Beta version": "No", + "Last Supported Version": "1.0.1", + "Cloud Native": "No" + }, + { + "Package name": "Aiohttp", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Asynqp", + "Support Policy": "Deprecated", + "Beta version": "No", + "Last Supported Version": "0.6", + "Cloud Native": "No" + }, + { + "Package name": "Boto3", + "Support Policy": "0-day", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Google-cloud-pubsub", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Google-cloud-storage", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Grpcio", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Mysqlclient", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Pika", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "PyMySQL", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Pymongo", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Psycopg2", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "No" + }, + { + "Package name": "Redis", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Requests", + "Support Policy": "0-day", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "SQLAlchemy", + "Support Policy": "30-days", + "Beta version": "No", + "Cloud Native": "Yes" + }, + { + "Package name": "Urllib3", + "Support Policy": "0-day", + "Beta version": "No", + "Cloud Native": "No" + } + ] + } diff --git a/.tekton/.currency/utils/tekton-ci-output.txt b/.tekton/.currency/resources/tekton-ci-output.txt similarity index 100% rename from .tekton/.currency/utils/tekton-ci-output.txt rename to .tekton/.currency/resources/tekton-ci-output.txt diff --git a/.tekton/.currency/scripts/generate_report.py b/.tekton/.currency/scripts/generate_report.py index a0a2597c..b043633c 100644 --- a/.tekton/.currency/scripts/generate_report.py +++ b/.tekton/.currency/scripts/generate_report.py @@ -10,7 +10,7 @@ from bs4 import BeautifulSoup -JSON_FILE = "utils/table.json" +JSON_FILE = "resources/table.json" REPORT_FILE = "docs/report.md" TEKTON_CI_OUT_FILE = "utils/tekton-ci-output.txt" TEKTON_CI_OUT_SCRIPT = "scripts/get-tekton-ci-output.sh" diff --git a/.tekton/.currency/scripts/get-tekton-ci-output.sh b/.tekton/.currency/scripts/get-tekton-ci-output.sh index 4be03a11..b3ae04ca 100644 --- a/.tekton/.currency/scripts/get-tekton-ci-output.sh +++ b/.tekton/.currency/scripts/get-tekton-ci-output.sh @@ -1,6 +1,6 @@ #!/bin/bash -TEKTON_CI_OUT_FILE=utils/tekton-ci-output.txt +TEKTON_CI_OUT_FILE=resources/tekton-ci-output.txt successful_taskruns=( $(kubectl get taskrun --sort-by=.metadata.creationTimestamp | grep "^python-trace\w*-unittest-default-3" | grep -v "pr\|Failed" | awk '{print $1}') ) diff --git a/.tekton/.currency/utils/table.json b/.tekton/.currency/utils/table.json deleted file mode 100644 index 8e6b773c..00000000 --- a/.tekton/.currency/utils/table.json +++ /dev/null @@ -1,165 +0,0 @@ -{ - "table": [ - { - "package_name": "ASGI", - "support_policy": "0-day", - "beta_version": "No", - "last_supported_version": "3.0", - "cloud_native": "No" - }, - { - "package_name": "Celery", - "support_policy": "30-days", - "beta_version": "No", - "cloud_native": "No" - }, - { - "package_name": "Django", - "support_policy": "30-days", - "beta_version": "No", - "cloud_native": "No" - }, - { - "package_name": "FastAPI", - "support_policy": "0-day", - "beta_version": "No", - "cloud_native": "No" - }, - { - "package_name": "Flask", - "support_policy": "0-day", - "beta_version": "No", - "cloud_native": "No" - }, - { - "package_name": "Pyramid", - "support_policy": "30-days", - "beta_version": "No", - "cloud_native": "No" - }, - { - "package_name": "Sanic", - "support_policy": "On demand", - "beta_version": "No", - "cloud_native": "No" - }, - { - "package_name": "Starlette", - "support_policy": "30-days", - "beta_version": "No", - "cloud_native": "No" - }, - { - "package_name": "Tornado", - "support_policy": "30-days", - "beta_version": "No", - "last_supported_version": "5.1.1", - "cloud_native": "No" - }, - { - "package_name": "Webapp2", - "support_policy": "On demand", - "beta_version": "No", - "last_supported_version": "2.5.2", - "cloud_native": "No" - }, - { - "package_name": "WSGI", - "support_policy": "0-day", - "beta_version": "No", - "last_supported_version": "1.0.1", - "cloud_native": "No" - }, - { - "package_name": "Aiohttp", - "support_policy": "30-days", - "beta_version": "No", - "cloud_native": "No" - }, - { - "package_name": "Asynqp", - "support_policy": "Deprecated", - "beta_version": "No", - "last_supported_version": "0.6", - "cloud_native": "No" - }, - { - "package_name": "Boto3", - "support_policy": "0-day", - "beta_version": "No", - "cloud_native": "Yes" - }, - { - "package_name": "Google-cloud-pubsub", - "support_policy": "30-days", - "beta_version": "No", - "cloud_native": "Yes" - }, - { - "package_name": "Google-cloud-storage", - "support_policy": "30-days", - "beta_version": "No", - "cloud_native": "Yes" - }, - { - "package_name": "Grpcio", - "support_policy": "30-days", - "beta_version": "No", - "cloud_native": "Yes" - }, - { - "package_name": "Mysqlclient", - "support_policy": "30-days", - "beta_version": "No", - "cloud_native": "Yes" - }, - { - "package_name": "Pika", - "support_policy": "30-days", - "beta_version": "No", - "cloud_native": "No" - }, - { - "package_name": "PyMySQL", - "support_policy": "30-days", - "beta_version": "No", - "cloud_native": "Yes" - }, - { - "package_name": "Pymongo", - "support_policy": "30-days", - "beta_version": "No", - "cloud_native": "Yes" - }, - { - "package_name": "Psycopg2", - "support_policy": "30-days", - "beta_version": "No", - "cloud_native": "No" - }, - { - "package_name": "Redis", - "support_policy": "30-days", - "beta_version": "No", - "cloud_native": "Yes" - }, - { - "package_name": "Requests", - "support_policy": "0-day", - "beta_version": "No", - "cloud_native": "Yes" - }, - { - "package_name": "SQLAlchemy", - "support_policy": "30-days", - "beta_version": "No", - "cloud_native": "Yes" - }, - { - "package_name": "Urllib3", - "support_policy": "0-day", - "beta_version": "No", - "cloud_native": "No" - } - ] - } From 69bb3e26a36304ba557153b3a6178fbdc8468ac7 Mon Sep 17 00:00:00 2001 From: Varsha GS Date: Sun, 12 May 2024 21:47:08 +0530 Subject: [PATCH 4/9] currency: use python kubernetes client to interact with the tekton cluster Signed-off-by: Varsha GS --- .tekton/.currency/currency-tasks.yaml | 5 - .../.currency/resources/tekton-ci-output.txt | 3 - .tekton/.currency/scripts/generate_report.py | 138 +++++++++++------- .../.currency/scripts/get-tekton-ci-output.sh | 16 -- 4 files changed, 87 insertions(+), 75 deletions(-) delete mode 100644 .tekton/.currency/resources/tekton-ci-output.txt delete mode 100644 .tekton/.currency/scripts/get-tekton-ci-output.sh diff --git a/.tekton/.currency/currency-tasks.yaml b/.tekton/.currency/currency-tasks.yaml index 458b03fa..5d8dc7e1 100644 --- a/.tekton/.currency/currency-tasks.yaml +++ b/.tekton/.currency/currency-tasks.yaml @@ -35,11 +35,6 @@ spec: image: python@sha256:c970ff53939772f47b0672e380328afb50d8fd1c0568ed4f82c22effc54244fc script: | #!/usr/bin/env bash - /usr/bin/curl -LO https://storage.googleapis.com/kubernetes-release/release/$(/usr/bin/curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ - chmod +x ./kubectl && \ - mv ./kubectl /usr/local/bin/kubectl - kubectl version - cd /workspace/python-sensor/.tekton/.currency python -m venv /tmp/venv diff --git a/.tekton/.currency/resources/tekton-ci-output.txt b/.tekton/.currency/resources/tekton-ci-output.txt deleted file mode 100644 index 025db350..00000000 --- a/.tekton/.currency/resources/tekton-ci-output.txt +++ /dev/null @@ -1,3 +0,0 @@ -[unittest] Successfully installed pip-24.0 -[unittest] Successfully installed autowrapt-1.0 basictracer-3.2.0 certifi-2024.2.2 charset-normalizer-3.3.2 fysom-2.1.6 idna-3.7 instana-2.3.0 opentracing-2.4.0 protobuf-4.25.3 requests-2.31.0 six-1.16.0 urllib3-2.2.1 wrapt-1.16.0 -[unittest] Successfully installed Django-5.0.4 Jinja2-3.1.3 PasteDeploy-3.1.0 PyMySQL-1.1.0 Werkzeug-3.0.2 aiofiles-23.2.1 aiohttp-3.9.5 aiosignal-1.3.1 amqp-5.2.0 annotated-types-0.6.0 anyio-4.3.0 asgiref-3.8.1 async-timeout-4.0.3 attrs-23.2.0 billiard-4.2.0 blinker-1.7.0 boto3-1.34.88 botocore-1.34.88 cachetools-5.3.3 celery-5.4.0 cffi-1.16.0 click-8.1.7 click-didyoumean-0.3.1 click-plugins-1.1.1 click-repl-0.3.0 coverage-7.4.4 cryptography-42.0.5 dnspython-2.6.1 exceptiongroup-1.2.1 fastapi-0.110.2 flask-3.0.3 frozenlist-1.4.1 google-api-core-1.34.1 google-auth-2.29.0 google-cloud-core-2.4.1 google-cloud-pubsub-2.1.0 google-cloud-storage-2.14.0 google-crc32c-1.5.0 google-resumable-media-2.7.0 googleapis-common-protos-1.63.0 greenlet-3.0.3 grpc-google-iam-v1-0.12.7 grpcio-1.62.2 grpcio-status-1.48.2 h11-0.14.0 httptools-0.6.1 hupper-1.12.1 iniconfig-2.0.0 itsdangerous-2.2.0 jmespath-1.0.1 kombu-5.3.7 libcst-1.3.1 lxml-5.2.1 markupsafe-2.1.5 mock-5.1.0 moto-5.0.5 multidict-5.2.0 mysqlclient-2.2.4 packaging-24.0 pika-1.3.2 plaster-1.1.2 plaster-pastedeploy-1.0.1 pluggy-1.5.0 prompt-toolkit-3.0.43 proto-plus-1.23.0 protobuf-3.20.3 psycopg2-binary-2.9.9 pyasn1-0.6.0 pyasn1-modules-0.4.0 pycparser-2.22 pydantic-2.7.0 pydantic-core-2.18.1 pymongo-4.6.3 pyramid-2.0.2 pytest-8.1.1 python-dateutil-2.9.0.post0 pytz-2024.1 pyyaml-6.0.1 redis-5.0.3 requests-mock-1.12.1 responses-0.17.0 rsa-4.9 s3transfer-0.10.1 sanic-21.6.2 sanic-routing-0.7.2 sniffio-1.3.1 spyne-2.14.0 sqlalchemy-2.0.29 sqlparse-0.5.0 starlette-0.37.2 tomli-2.0.1 translationstring-1.4 typing-extensions-4.11.0 tzdata-2024.1 ujson-5.9.0 uvicorn-0.29.0 uvloop-0.19.0 venusian-3.1.0 vine-5.1.0 wcwidth-0.2.13 webob-1.8.7 websockets-12.0 xmltodict-0.13.0 yarl-1.9.4 zope.deprecation-5.0 zope.interface-6.3 diff --git a/.tekton/.currency/scripts/generate_report.py b/.tekton/.currency/scripts/generate_report.py index b043633c..e21be521 100644 --- a/.tekton/.currency/scripts/generate_report.py +++ b/.tekton/.currency/scripts/generate_report.py @@ -1,19 +1,16 @@ # Standard Libraries import re import json -from os import system from datetime import date # Third Party import requests import pandas as pd from bs4 import BeautifulSoup - +from kubernetes import client, config JSON_FILE = "resources/table.json" REPORT_FILE = "docs/report.md" -TEKTON_CI_OUT_FILE = "utils/tekton-ci-output.txt" -TEKTON_CI_OUT_SCRIPT = "scripts/get-tekton-ci-output.sh" PIP_INDEX_URL = "https://pypi.org/pypi" SPEC_MAP = { @@ -55,21 +52,16 @@ def get_upstream_version(dependency): return latest_version -## Get the tekton ci output of the installed python dependencies -system("bash " + TEKTON_CI_OUT_SCRIPT) - -with open(TEKTON_CI_OUT_FILE) as file: - content = file.read() - - -def get_last_supported_version(dependency): +def get_last_supported_version(tekton_ci_output, dependency): """get up-to-date supported version""" pattern = r"-([^\s]+)" if dependency == "Psycopg2": dependency = "psycopg2-binary" - last_supported_version = re.search(dependency + pattern, content, flags=re.I | re.M) + last_supported_version = re.search( + dependency + pattern, tekton_ci_output, flags=re.I | re.M + ) return last_supported_version[1] @@ -83,56 +75,100 @@ def isUptodate(last_supported_version, latest_version): return up_to_date -# Read the JSON file -with open(JSON_FILE) as file: - data = json.load(file) +def get_tekton_ci_output(): + # config.load_kube_config() + config.load_incluster_config() + + group = "tekton.dev" + version = "v1" + namespace = "default" + plural = "taskruns" + + # access the custom resource from tekton + tektonV1 = client.CustomObjectsApi() + taskruns = tektonV1.list_namespaced_custom_object( + group, + version, + namespace, + plural, + label_selector=f"{group}/task=python-tracer-unittest-default-task", + )["items"] + + taskruns.sort(key=lambda tr: tr["metadata"]["creationTimestamp"], reverse=True) + + coreV1 = client.CoreV1Api() + tekton_ci_output = "" + for tr in taskruns: + if ( + re.match("python-trace\w+-unittest-default-3", tr["metadata"]["name"]) + and tr["status"]["conditions"][0]["type"] == "Succeeded" + ): + pod = tr["status"]["podName"] + logs = coreV1.read_namespaced_pod_log( + pod, namespace, container="step-unittest" + ) + if "Successfully installed" in logs: + for line in logs.splitlines(): + if "Successfully installed" in line: + tekton_ci_output += line + break + return tekton_ci_output + +def main(): + # Read the JSON file + with open(JSON_FILE) as file: + data = json.load(file) -items = data["table"] + items = data["table"] + tekton_ci_output = get_tekton_ci_output() -for index in range(len(items)): - item = items[index] - package = item["package_name"] + for item in items: + package = item["Package name"] - if "last_supported_version" not in item: - last_supported_version = get_last_supported_version(package) - item.update({"last_supported_version": last_supported_version}) - else: - last_supported_version = item["last_supported_version"] + if "Last Supported Version" not in item: + last_supported_version = get_last_supported_version( + tekton_ci_output, package + ) + item.update({"Last Supported Version": last_supported_version}) + else: + last_supported_version = item["Last Supported Version"] + + latest_version = get_upstream_version(package) - latest_version = get_upstream_version(package) - item.update({"latest_version": latest_version}) + up_to_date = isUptodate(last_supported_version, latest_version) - up_to_date = isUptodate(last_supported_version, latest_version) + item.update({"Latest version": latest_version, "Up-to-date": up_to_date}) - item.update({"up_to_date": up_to_date}) + # Create a DataFrame from the list of dictionaries + df = pd.DataFrame(items) + df.insert(len(df.columns) - 1, "Cloud Native", df.pop("Cloud Native")) + # Rename Columns + df.columns = [ + "Package name", + "Support Policy", + "Beta version", + "Last Supported Version", + "Latest version", + "Up-to-date", + "Cloud Native", + ] -# Create a DataFrame from the list of dictionaries -df = pd.DataFrame(items) -df.insert(len(df.columns) - 1, "cloud_native", df.pop("cloud_native")) + # Convert dataframe to markdown + markdown_table = df.to_markdown(index=False) -# Rename Columns -df.columns = [ - "Package name", - "Support Policy", - "Beta version", - "Last Supported Version", - "Latest version", - "Up-to-date", - "Cloud Native", -] + current_date = date.today().strftime("%b %d, %Y") -# Convert dataframe to markdown -markdown_table = df.to_markdown(index=False) + disclaimer = f"##### This page is auto-generated. Any change will be overwritten after the next sync. Please apply changes directly to the files in the [python tracer](https://github.com/instana/python-sensor) repo. Last updated on **{current_date}**." + title = "## Python supported packages and versions" -current_date = date.today().strftime("%b %d, %Y") + # Combine disclaimer, title, and markdown table with line breaks + final_markdown = disclaimer + "\n" + title + "\n" + markdown_table -disclaimer = f"This page is auto-generated. Any change will be overwritten after the next sync. Please apply changes directly to the files in the [python tracer](https://github.com/instana/python-sensor) repo. Last updated on **{current_date}**." -title = "## Python supported packages and versions" + with open(REPORT_FILE, "w") as file: + file.write(final_markdown) -# Combine disclaimer, title, and markdown table with line breaks -final_markdown = disclaimer + "\n" + title + "\n" + markdown_table -with open(REPORT_FILE, "w") as file: - file.write(final_markdown) +if __name__ == "__main__": + main() diff --git a/.tekton/.currency/scripts/get-tekton-ci-output.sh b/.tekton/.currency/scripts/get-tekton-ci-output.sh deleted file mode 100644 index b3ae04ca..00000000 --- a/.tekton/.currency/scripts/get-tekton-ci-output.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -TEKTON_CI_OUT_FILE=resources/tekton-ci-output.txt - -successful_taskruns=( $(kubectl get taskrun --sort-by=.metadata.creationTimestamp | grep "^python-trace\w*-unittest-default-3" | grep -v "pr\|Failed" | awk '{print $1}') ) - -for ((i=${#successful_taskruns[@]}-1; i>=0; i--)); do - pod_name=$(kubectl get taskrun "${successful_taskruns[$i]}" -o jsonpath='{.status.podName}') - ci_output=$(kubectl logs ${pod_name} -c step-unittest | grep "Successfully installed") - if [ -n "${ci_output}" ]; then - latest_successful_taskrun_pod=$pod_name - break - fi -done - -kubectl logs ${latest_successful_taskrun_pod} -c step-unittest | grep "Successfully installed" > ${TEKTON_CI_OUT_FILE} From f9103637a839a698c62fcae2de39d34f16d6c904 Mon Sep 17 00:00:00 2001 From: Varsha GS Date: Mon, 13 May 2024 14:00:20 +0530 Subject: [PATCH 5/9] chore(currency): change revision to master, upload report only on successful git clone Signed-off-by: Varsha GS --- .tekton/.currency/currency-pipelinerun.yaml | 2 +- .../currency-scheduled-eventlistener.yaml | 2 +- .tekton/.currency/currency-tasks.yaml | 20 ++++++++++--------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.tekton/.currency/currency-pipelinerun.yaml b/.tekton/.currency/currency-pipelinerun.yaml index a0c2e162..151f5403 100644 --- a/.tekton/.currency/currency-pipelinerun.yaml +++ b/.tekton/.currency/currency-pipelinerun.yaml @@ -5,7 +5,7 @@ metadata: spec: params: - name: revision - value: "currency-update" + value: "master" pipelineRef: name: python-currency-pipeline serviceAccountName: currency-serviceaccount diff --git a/.tekton/.currency/currency-scheduled-eventlistener.yaml b/.tekton/.currency/currency-scheduled-eventlistener.yaml index 50d49c95..a7916e30 100644 --- a/.tekton/.currency/currency-scheduled-eventlistener.yaml +++ b/.tekton/.currency/currency-scheduled-eventlistener.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: currency-serviceaccount params: - name: revision - value: "currency-update" + value: "master" workspaces: - name: currency-pvc volumeClaimTemplate: diff --git a/.tekton/.currency/currency-tasks.yaml b/.tekton/.currency/currency-tasks.yaml index 5d8dc7e1..f9a3ba61 100644 --- a/.tekton/.currency/currency-tasks.yaml +++ b/.tekton/.currency/currency-tasks.yaml @@ -70,14 +70,16 @@ spec: cd /workspace git clone https://oauth2:$GH_ENTERPRISE_TOKEN@github.ibm.com/instana/tracer-reports.git - cd tracer-reports + if [ $? -eq 0 ]; then + cd tracer-reports - cp ../python-sensor/.tekton/.currency/docs/report.md ./automated/currency/python/report.md + cp ../python-sensor/.tekton/.currency/docs/report.md ./automated/currency/python/report.md - git config user.name "Instanacd PAT for GitHub Enterprise" - git config user.email instana.ibm.github.enterprise@ibm.com - - git add . - - git commit -m "chore: Updated python currency report" - git push origin main + git config user.name "Instanacd PAT for GitHub Enterprise" + git config user.email instana.ibm.github.enterprise@ibm.com + + git add . + + git commit -m "chore: Updated python currency report" + git push origin main + fi From 74511da3a480f71438f475f1d50b36e2ed833466 Mon Sep 17 00:00:00 2001 From: Varsha GS Date: Tue, 14 May 2024 10:02:29 +0530 Subject: [PATCH 6/9] currency: extract starlette version from python-tracer-unittest-gevent-starlette-task Signed-off-by: Varsha GS --- .tekton/.currency/scripts/generate_report.py | 56 ++++++++++++-------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/.tekton/.currency/scripts/generate_report.py b/.tekton/.currency/scripts/generate_report.py index e21be521..81d28858 100644 --- a/.tekton/.currency/scripts/generate_report.py +++ b/.tekton/.currency/scripts/generate_report.py @@ -1,7 +1,6 @@ # Standard Libraries import re import json -from datetime import date # Third Party import requests @@ -75,13 +74,9 @@ def isUptodate(last_supported_version, latest_version): return up_to_date -def get_tekton_ci_output(): - # config.load_kube_config() - config.load_incluster_config() - +def get_taskruns(namespace, task): group = "tekton.dev" version = "v1" - namespace = "default" plural = "taskruns" # access the custom resource from tekton @@ -91,16 +86,44 @@ def get_tekton_ci_output(): version, namespace, plural, - label_selector=f"{group}/task=python-tracer-unittest-default-task", + label_selector=f"{group}/task={task}, triggers.tekton.dev/trigger=python-tracer-scheduled-pipeline-triggger", )["items"] taskruns.sort(key=lambda tr: tr["metadata"]["creationTimestamp"], reverse=True) + return taskruns + + +def get_tekton_ci_output(): + # config.load_kube_config() + config.load_incluster_config() + + namespace = "default" + + starlette_taskruns = get_taskruns( + namespace, task="python-tracer-unittest-gevent-starlette-task" + ) + coreV1 = client.CoreV1Api() tekton_ci_output = "" - for tr in taskruns: + for tr in starlette_taskruns: + if tr["status"]["conditions"][0]["type"] == "Succeeded": + pod = tr["status"]["podName"] + logs = coreV1.read_namespaced_pod_log( + pod, namespace, container="step-unittest" + ) + if "Successfully installed" in logs: + match = re.search("Successfully installed .* (starlette-[^\s]+)", logs) + tekton_ci_output += f"{match[1]}\n" + break + + default_taskruns = get_taskruns( + namespace, task="python-tracer-unittest-default-task" + ) + + for tr in default_taskruns: if ( - re.match("python-trace\w+-unittest-default-3", tr["metadata"]["name"]) + tr["metadata"]["name"].endswith("unittest-default-3") and tr["status"]["conditions"][0]["type"] == "Succeeded" ): pod = tr["status"]["podName"] @@ -144,23 +167,10 @@ def main(): df = pd.DataFrame(items) df.insert(len(df.columns) - 1, "Cloud Native", df.pop("Cloud Native")) - # Rename Columns - df.columns = [ - "Package name", - "Support Policy", - "Beta version", - "Last Supported Version", - "Latest version", - "Up-to-date", - "Cloud Native", - ] - # Convert dataframe to markdown markdown_table = df.to_markdown(index=False) - current_date = date.today().strftime("%b %d, %Y") - - disclaimer = f"##### This page is auto-generated. Any change will be overwritten after the next sync. Please apply changes directly to the files in the [python tracer](https://github.com/instana/python-sensor) repo. Last updated on **{current_date}**." + disclaimer = f"##### This page is auto-generated. Any change will be overwritten after the next sync. Please apply changes directly to the files in the [python tracer](https://github.com/instana/python-sensor) repo." title = "## Python supported packages and versions" # Combine disclaimer, title, and markdown table with line breaks From 1a7d2d5af3058aa55ae3c9504c4fc9e4600946d0 Mon Sep 17 00:00:00 2001 From: Varsha GS Date: Mon, 20 May 2024 20:39:12 +0530 Subject: [PATCH 7/9] currency: filter taskruns before sorting Signed-off-by: Varsha GS --- .tekton/.currency/currency-rbac.yaml | 4 +- .tekton/.currency/currency-tasks.yaml | 26 ++++--- .tekton/.currency/scripts/generate_report.py | 76 ++++++++++++-------- 3 files changed, 64 insertions(+), 42 deletions(-) diff --git a/.tekton/.currency/currency-rbac.yaml b/.tekton/.currency/currency-rbac.yaml index b0b32765..aca210e4 100644 --- a/.tekton/.currency/currency-rbac.yaml +++ b/.tekton/.currency/currency-rbac.yaml @@ -10,10 +10,10 @@ metadata: rules: - apiGroups: [""] resources: ["pods", "pods/log"] - verbs: ["get", "list", "watch"] + verbs: ["get", "list"] - apiGroups: ["tekton.dev"] resources: ["taskruns"] - verbs: ["get", "list", "watch"] + verbs: ["get", "list"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/.tekton/.currency/currency-tasks.yaml b/.tekton/.currency/currency-tasks.yaml index f9a3ba61..6a43f7e1 100644 --- a/.tekton/.currency/currency-tasks.yaml +++ b/.tekton/.currency/currency-tasks.yaml @@ -70,16 +70,20 @@ spec: cd /workspace git clone https://oauth2:$GH_ENTERPRISE_TOKEN@github.ibm.com/instana/tracer-reports.git - if [ $? -eq 0 ]; then - cd tracer-reports - cp ../python-sensor/.tekton/.currency/docs/report.md ./automated/currency/python/report.md - - git config user.name "Instanacd PAT for GitHub Enterprise" - git config user.email instana.ibm.github.enterprise@ibm.com - - git add . - - git commit -m "chore: Updated python currency report" - git push origin main + if [ $? -ne 0 ]; then + echo "The attempt to clone the tracer-reports repository failed, preventing the upload of python tracer currency report." >&2 + exit 1 fi + + cd tracer-reports + + cp ../python-sensor/.tekton/.currency/docs/report.md ./automated/currency/python/report.md + + git config user.name "Instanacd PAT for GitHub Enterprise" + git config user.email instana.ibm.github.enterprise@ibm.com + + git add . + + git commit -m "chore: Updated python currency report" + git push origin main diff --git a/.tekton/.currency/scripts/generate_report.py b/.tekton/.currency/scripts/generate_report.py index 81d28858..0a1f107b 100644 --- a/.tekton/.currency/scripts/generate_report.py +++ b/.tekton/.currency/scripts/generate_report.py @@ -74,7 +74,7 @@ def isUptodate(last_supported_version, latest_version): return up_to_date -def get_taskruns(namespace, task): +def get_taskruns(namespace, task_name, taskrun_filter): group = "tekton.dev" version = "v1" plural = "taskruns" @@ -86,12 +86,15 @@ def get_taskruns(namespace, task): version, namespace, plural, - label_selector=f"{group}/task={task}, triggers.tekton.dev/trigger=python-tracer-scheduled-pipeline-triggger", + label_selector=f"{group}/task={task_name}, triggers.tekton.dev/trigger=python-tracer-scheduled-pipeline-triggger", )["items"] - taskruns.sort(key=lambda tr: tr["metadata"]["creationTimestamp"], reverse=True) + filtered_taskruns = list(filter(taskrun_filter, taskruns)) + filtered_taskruns.sort( + key=lambda tr: tr["metadata"]["creationTimestamp"], reverse=True + ) - return taskruns + return filtered_taskruns def get_tekton_ci_output(): @@ -100,41 +103,56 @@ def get_tekton_ci_output(): namespace = "default" - starlette_taskruns = get_taskruns( - namespace, task="python-tracer-unittest-gevent-starlette-task" - ) + task_name = "python-tracer-unittest-gevent-starlette-task" + taskrun_filter = lambda tr: tr["status"]["conditions"][0]["type"] == "Succeeded" + + starlette_taskruns = get_taskruns(namespace, task_name, taskrun_filter) coreV1 = client.CoreV1Api() tekton_ci_output = "" for tr in starlette_taskruns: - if tr["status"]["conditions"][0]["type"] == "Succeeded": - pod = tr["status"]["podName"] - logs = coreV1.read_namespaced_pod_log( - pod, namespace, container="step-unittest" + pod_name = tr["status"]["podName"] + taskrun_name = tr["metadata"]["name"] + logs = coreV1.read_namespaced_pod_log( + pod_name, namespace, container="step-unittest" + ) + if "Successfully installed" in logs: + print( + f"Retrieving container logs from the successful taskrun pod {pod_name} of taskrun {taskrun_name}.." + ) + match = re.search("Successfully installed .* (starlette-[^\s]+)", logs) + tekton_ci_output += f"{match[1]}\n" + break + else: + print( + f"Unable to retrieve container logs from the successful taskrun pod {pod_name} of taskrun {taskrun_name}." ) - if "Successfully installed" in logs: - match = re.search("Successfully installed .* (starlette-[^\s]+)", logs) - tekton_ci_output += f"{match[1]}\n" - break - default_taskruns = get_taskruns( - namespace, task="python-tracer-unittest-default-task" + task_name = "python-tracer-unittest-default-task" + taskrun_filter = ( + lambda tr: tr["metadata"]["name"].endswith("unittest-default-3") + and tr["status"]["conditions"][0]["type"] == "Succeeded" ) + default_taskruns = get_taskruns(namespace, task_name, taskrun_filter) for tr in default_taskruns: - if ( - tr["metadata"]["name"].endswith("unittest-default-3") - and tr["status"]["conditions"][0]["type"] == "Succeeded" - ): - pod = tr["status"]["podName"] - logs = coreV1.read_namespaced_pod_log( - pod, namespace, container="step-unittest" + pod_name = tr["status"]["podName"] + taskrun_name = tr["metadata"]["name"] + logs = coreV1.read_namespaced_pod_log( + pod_name, namespace, container="step-unittest" + ) + if "Successfully installed" in logs: + print( + f"Retrieving container logs from the successful taskrun pod {pod_name} of taskrun {taskrun_name}.." + ) + for line in logs.splitlines(): + if "Successfully installed" in line: + tekton_ci_output += line + break + else: + print( + f"Unable to retrieve container logs from the successful taskrun pod {pod_name} of taskrun {taskrun_name}." ) - if "Successfully installed" in logs: - for line in logs.splitlines(): - if "Successfully installed" in line: - tekton_ci_output += line - break return tekton_ci_output From 9593c8c1d700d28fd10de3458c55f896d2ae7a26 Mon Sep 17 00:00:00 2001 From: Varsha GS Date: Wed, 22 May 2024 12:49:21 +0530 Subject: [PATCH 8/9] chore(currency): make processing taskrun logs reusable Signed-off-by: Varsha GS --- .tekton/.currency/scripts/generate_report.py | 68 ++++++++++---------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/.tekton/.currency/scripts/generate_report.py b/.tekton/.currency/scripts/generate_report.py index 0a1f107b..f8b9bb34 100644 --- a/.tekton/.currency/scripts/generate_report.py +++ b/.tekton/.currency/scripts/generate_report.py @@ -97,36 +97,48 @@ def get_taskruns(namespace, task_name, taskrun_filter): return filtered_taskruns -def get_tekton_ci_output(): - # config.load_kube_config() - config.load_incluster_config() - - namespace = "default" - - task_name = "python-tracer-unittest-gevent-starlette-task" - taskrun_filter = lambda tr: tr["status"]["conditions"][0]["type"] == "Succeeded" - - starlette_taskruns = get_taskruns(namespace, task_name, taskrun_filter) - - coreV1 = client.CoreV1Api() - tekton_ci_output = "" - for tr in starlette_taskruns: +def process_taskrun_logs( + taskruns, core_v1_client, namespace, task_name, tekton_ci_output +): + for tr in taskruns: pod_name = tr["status"]["podName"] taskrun_name = tr["metadata"]["name"] - logs = coreV1.read_namespaced_pod_log( + logs = core_v1_client.read_namespaced_pod_log( pod_name, namespace, container="step-unittest" ) if "Successfully installed" in logs: print( f"Retrieving container logs from the successful taskrun pod {pod_name} of taskrun {taskrun_name}.." ) - match = re.search("Successfully installed .* (starlette-[^\s]+)", logs) - tekton_ci_output += f"{match[1]}\n" + if task_name == "python-tracer-unittest-gevent-starlette-task": + match = re.search("Successfully installed .* (starlette-[^\s]+)", logs) + tekton_ci_output += f"{match[1]}\n" + elif task_name == "python-tracer-unittest-default-task": + for line in logs.splitlines(): + if "Successfully installed" in line: + tekton_ci_output += line break else: print( f"Unable to retrieve container logs from the successful taskrun pod {pod_name} of taskrun {taskrun_name}." ) + return tekton_ci_output + + +def get_tekton_ci_output(): + # config.load_kube_config() + config.load_incluster_config() + + namespace = "default" + core_v1_client = client.CoreV1Api() + + task_name = "python-tracer-unittest-gevent-starlette-task" + taskrun_filter = lambda tr: tr["status"]["conditions"][0]["type"] == "Succeeded" + starlette_taskruns = get_taskruns(namespace, task_name, taskrun_filter) + + tekton_ci_output = process_taskrun_logs( + starlette_taskruns, core_v1_client, namespace, task_name, "" + ) task_name = "python-tracer-unittest-default-task" taskrun_filter = ( @@ -135,24 +147,10 @@ def get_tekton_ci_output(): ) default_taskruns = get_taskruns(namespace, task_name, taskrun_filter) - for tr in default_taskruns: - pod_name = tr["status"]["podName"] - taskrun_name = tr["metadata"]["name"] - logs = coreV1.read_namespaced_pod_log( - pod_name, namespace, container="step-unittest" - ) - if "Successfully installed" in logs: - print( - f"Retrieving container logs from the successful taskrun pod {pod_name} of taskrun {taskrun_name}.." - ) - for line in logs.splitlines(): - if "Successfully installed" in line: - tekton_ci_output += line - break - else: - print( - f"Unable to retrieve container logs from the successful taskrun pod {pod_name} of taskrun {taskrun_name}." - ) + tekton_ci_output = process_taskrun_logs( + default_taskruns, core_v1_client, namespace, task_name, tekton_ci_output + ) + return tekton_ci_output From 5fa0ef99e7d3f18f26a82846d8698ac58c08b38e Mon Sep 17 00:00:00 2001 From: Varsha GS Date: Fri, 24 May 2024 22:08:12 +0530 Subject: [PATCH 9/9] chore(currency): - add docstrings - uploadlatest currency report Signed-off-by: Varsha GS --- .tekton/.currency/docs/report.md | 20 +++++++++++--------- .tekton/.currency/scripts/generate_report.py | 8 ++++++-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.tekton/.currency/docs/report.md b/.tekton/.currency/docs/report.md index f7507415..7ff39513 100644 --- a/.tekton/.currency/docs/report.md +++ b/.tekton/.currency/docs/report.md @@ -1,9 +1,11 @@ +##### This page is auto-generated. Any change will be overwritten after the next sync. Please apply changes directly to the files in the [python tracer](https://github.com/instana/python-sensor) repo. +## Python supported packages and versions | Package name | Support Policy | Beta version | Last Supported Version | Latest version | Up-to-date | Cloud Native | |:---------------------|:-----------------|:---------------|:-------------------------|:-----------------|:-------------|:---------------| | ASGI | 0-day | No | 3.0 | 3.0 | Yes | No | | Celery | 30-days | No | 5.4.0 | 5.4.0 | Yes | No | -| Django | 30-days | No | 5.0.4 | 5.0.4 | Yes | No | -| FastAPI | 0-day | No | 0.110.2 | 0.110.2 | Yes | No | +| Django | 30-days | No | 5.0.6 | 5.0.6 | Yes | No | +| FastAPI | 0-day | No | 0.111.0 | 0.111.0 | Yes | No | | Flask | 0-day | No | 3.0.3 | 3.0.3 | Yes | No | | Pyramid | 30-days | No | 2.0.2 | 2.0.2 | Yes | No | | Sanic | On demand | No | 21.6.2 | 23.12.1 | No | No | @@ -13,16 +15,16 @@ | WSGI | 0-day | No | 1.0.1 | 1.0.1 | Yes | No | | Aiohttp | 30-days | No | 3.9.5 | 3.9.5 | Yes | No | | Asynqp | Deprecated | No | 0.6 | 0.6 | Yes | No | -| Boto3 | 0-day | No | 1.34.88 | 1.34.88 | Yes | Yes | +| Boto3 | 0-day | No | 1.34.112 | 1.34.112 | Yes | Yes | | Google-cloud-pubsub | 30-days | No | 2.1.0 | 2.21.1 | No | Yes | | Google-cloud-storage | 30-days | No | 2.14.0 | 2.16.0 | No | Yes | -| Grpcio | 30-days | No | 1.62.2 | 1.62.2 | Yes | Yes | +| Grpcio | 30-days | No | 1.64.0 | 1.64.0 | Yes | Yes | | Mysqlclient | 30-days | No | 2.2.4 | 2.2.4 | Yes | Yes | | Pika | 30-days | No | 1.3.2 | 1.3.2 | Yes | No | -| PyMySQL | 30-days | No | 1.1.0 | 1.1.0 | Yes | Yes | -| Pymongo | 30-days | No | 4.6.3 | 4.6.3 | Yes | Yes | +| PyMySQL | 30-days | No | 1.1.1 | 1.1.1 | Yes | Yes | +| Pymongo | 30-days | No | 4.7.2 | 4.7.2 | Yes | Yes | | Psycopg2 | 30-days | No | 2.9.9 | 2.9.9 | Yes | No | -| Redis | 30-days | No | 5.0.3 | 5.0.3 | Yes | Yes | -| Requests | 0-day | No | 2.31.0 | 2.31.0 | Yes | Yes | -| SQLAlchemy | 30-days | No | 2.0.29 | 2.0.29 | Yes | Yes | +| Redis | 30-days | No | 5.0.4 | 5.0.4 | Yes | Yes | +| Requests | 0-day | No | 2.32.2 | 2.32.2 | Yes | Yes | +| SQLAlchemy | 30-days | No | 2.0.30 | 2.0.30 | Yes | Yes | | Urllib3 | 0-day | No | 2.2.1 | 2.2.1 | Yes | No | \ No newline at end of file diff --git a/.tekton/.currency/scripts/generate_report.py b/.tekton/.currency/scripts/generate_report.py index f8b9bb34..2ae36597 100644 --- a/.tekton/.currency/scripts/generate_report.py +++ b/.tekton/.currency/scripts/generate_report.py @@ -19,7 +19,7 @@ def get_upstream_version(dependency): - """get the latest version available upstream""" + """Get the latest version available upstream""" if dependency in SPEC_MAP: # webscrape info from official website pattern = "(\d+\.\d+\.?\d*)" @@ -52,7 +52,7 @@ def get_upstream_version(dependency): def get_last_supported_version(tekton_ci_output, dependency): - """get up-to-date supported version""" + """Get up-to-date supported version""" pattern = r"-([^\s]+)" if dependency == "Psycopg2": @@ -66,6 +66,7 @@ def get_last_supported_version(tekton_ci_output, dependency): def isUptodate(last_supported_version, latest_version): + """Check if the supported package is up-to-date""" if last_supported_version == latest_version: up_to_date = "Yes" else: @@ -75,6 +76,7 @@ def isUptodate(last_supported_version, latest_version): def get_taskruns(namespace, task_name, taskrun_filter): + """Get sorted taskruns filtered based on label_selector""" group = "tekton.dev" version = "v1" plural = "taskruns" @@ -100,6 +102,7 @@ def get_taskruns(namespace, task_name, taskrun_filter): def process_taskrun_logs( taskruns, core_v1_client, namespace, task_name, tekton_ci_output ): + """Process taskrun logs""" for tr in taskruns: pod_name = tr["status"]["podName"] taskrun_name = tr["metadata"]["name"] @@ -126,6 +129,7 @@ def process_taskrun_logs( def get_tekton_ci_output(): + """Get the latest successful scheduled tekton pipeline output""" # config.load_kube_config() config.load_incluster_config()