From 0cb2202a8308ac4a94764f2a9fb1a7ab6c3bae64 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Tue, 2 Nov 2021 14:54:44 -0700 Subject: [PATCH 01/23] trying build status test --- infra/build/build_status/Dockerfile | 8 + infra/build/build_status/cloudbuild.yaml | 10 + .../build/build_status/update_build_status.py | 319 ++++++++++++++++++ 3 files changed, 337 insertions(+) create mode 100644 infra/build/build_status/Dockerfile create mode 100644 infra/build/build_status/cloudbuild.yaml create mode 100644 infra/build/build_status/update_build_status.py diff --git a/infra/build/build_status/Dockerfile b/infra/build/build_status/Dockerfile new file mode 100644 index 000000000000..7d88c110adb8 --- /dev/null +++ b/infra/build/build_status/Dockerfile @@ -0,0 +1,8 @@ +FROM gcr.io/oss-fuzz-base/base-runner + + +RUN mkdir -p /opt/oss-fuzz/infra/build_status +COPY ./functions/* /opt/oss-fuzz/infra/build_status/ +COPY ./build_status/* /opt/oss-fuzz/infra/build_status/ +RUN pip install -r /opt/oss-fuzz/infra/build_status/requirements.txt +ENTRYPOINT [ "python3", "/opt/oss-fuzz/infra/build_status/update_build_status.py" ] \ No newline at end of file diff --git a/infra/build/build_status/cloudbuild.yaml b/infra/build/build_status/cloudbuild.yaml new file mode 100644 index 000000000000..21e554713914 --- /dev/null +++ b/infra/build/build_status/cloudbuild.yaml @@ -0,0 +1,10 @@ +steps: +- name: 'gcr.io/cloud-builders/docker' + args: + - build + - '-t' + - gcr.io/oss-fuzz-base/build-status + - '-f' + - infra/build_status/Dockerfile +- name: 'gcr.io/cloud-builders/docker' + args: ['run', 'gcr.io/oss-fuzz-base/build-status'] \ No newline at end of file diff --git a/infra/build/build_status/update_build_status.py b/infra/build/build_status/update_build_status.py new file mode 100644 index 000000000000..a807c00d3b89 --- /dev/null +++ b/infra/build/build_status/update_build_status.py @@ -0,0 +1,319 @@ +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +"""Cloud function to request builds.""" +import base64 +import concurrent.futures +import json +import sys + +import google.auth +from googleapiclient.discovery import build +from google.cloud import ndb +from google.cloud import storage + +import build_and_run_coverage +import build_project +from datastore_entities import BuildsHistory +from datastore_entities import LastSuccessfulBuild +from datastore_entities import Project + +BADGE_DIR = 'badge_images' +BADGE_IMAGE_TYPES = {'svg': 'image/svg+xml', 'png': 'image/png'} +DESTINATION_BADGE_DIR = 'badges' +MAX_BUILD_LOGS = 7 + +STATUS_BUCKET = 'oss-fuzz-build-logs-test' + +FUZZING_STATUS_FILENAME = 'status.json' +COVERAGE_STATUS_FILENAME = 'status-coverage.json' + +# pylint: disable=invalid-name +_client = None + + +class MissingBuildLogError(Exception): + """Missing build log file in cloud storage.""" + + +# pylint: disable=global-statement +def get_storage_client(): + """Return storage client.""" + global _client + if not _client: + _client = storage.Client() + + return _client + + +def is_build_successful(build_obj): + """Check build success.""" + return build_obj['status'] == 'SUCCESS' + + +def upload_status(data, status_filename): + """Upload json file to cloud storage.""" + bucket = get_storage_client().get_bucket(STATUS_BUCKET) + blob = bucket.blob(status_filename) + blob.cache_control = 'no-cache' + blob.upload_from_string(json.dumps(data), content_type='application/json') + + +def sort_projects(projects): + """Sort projects in order Failures, Successes, Not yet built.""" + + def key_func(project): + if not project['history']: + return 2 # Order projects without history last. + + if project['history'][0]['success']: + # Successful builds come second. + return 1 + + # Build failures come first. + return 0 + + projects.sort(key=key_func) + + +def get_build(cloudbuild, image_project, build_id): + """Get build object from cloudbuild.""" + return cloudbuild.projects().builds().get(projectId=image_project, + id=build_id).execute() + + +def update_last_successful_build(project, build_tag): + """Update last successful build.""" + last_successful_build = ndb.Key(LastSuccessfulBuild, + project['name'] + '-' + build_tag).get() + if not last_successful_build and 'last_successful_build' not in project: + return + + if 'last_successful_build' not in project: + project['last_successful_build'] = { + 'build_id': last_successful_build.build_id, + 'finish_time': last_successful_build.finish_time + } + else: + if last_successful_build: + last_successful_build.build_id = project['last_successful_build'][ + 'build_id'] + last_successful_build.finish_time = project['last_successful_build'][ + 'finish_time'] + else: + last_successful_build = LastSuccessfulBuild( + id=project['name'] + '-' + build_tag, + project=project['name'], + build_id=project['last_successful_build']['build_id'], + finish_time=project['last_successful_build']['finish_time']) + last_successful_build.put() + + +# pylint: disable=no-member +def get_build_history(build_ids): + """Returns build object for the last finished build of project.""" + credentials, image_project = google.auth.default() + cloudbuild = build('cloudbuild', + 'v1', + credentials=credentials, + cache_discovery=False) + + history = [] + last_successful_build = None + + for build_id in reversed(build_ids): + project_build = get_build(cloudbuild, image_project, build_id) + if project_build['status'] not in ('SUCCESS', 'FAILURE', 'TIMEOUT'): + continue + + if (not last_successful_build and is_build_successful(project_build)): + last_successful_build = { + 'build_id': build_id, + 'finish_time': project_build['finishTime'], + } + + if not upload_log(build_id): + log_name = f'log-{build_id}' + raise MissingBuildLogError(f'Missing build log file {log_name}') + + history.append({ + 'build_id': build_id, + 'finish_time': project_build['finishTime'], + 'success': is_build_successful(project_build) + }) + + if len(history) == MAX_BUILD_LOGS: + break + + project = {'history': history} + if last_successful_build: + project['last_successful_build'] = last_successful_build + return project + + +# pylint: disable=too-many-locals +def update_build_status(build_tag, status_filename): + """Update build statuses.""" + projects = [] + + def process_project(project_build): + """Process a project.""" + project = get_build_history(project_build.build_ids) + project['name'] = project_build.project + print('Processing project', project['name']) + return project + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + futures = [] + for project_build in BuildsHistory.query( + BuildsHistory.build_tag == build_tag).order('project'): + futures.append(executor.submit(process_project, project_build)) + + for future in concurrent.futures.as_completed(futures): + project = future.result() + update_last_successful_build(project, build_tag) + projects.append(project) + + sort_projects(projects) + data = {'projects': projects} + upload_status(data, status_filename) + + +def update_build_badges(project, last_build_successful, + last_coverage_build_successful): + """Upload badges of given project.""" + badge = 'building' + # last_coverage_build_successful is False if there was an unsuccessful build + # and None if the target does not support coverage (e.g. Python or Java + # targets). + if last_coverage_build_successful is False: + badge = 'coverage_failing' + if not last_build_successful: + badge = 'failing' + + print(f'[badge] {project}: {badge}') + + for extension in BADGE_IMAGE_TYPES: + badge_name = f'{badge}.{extension}' + + # Copy blob from badge_images/badge_name to badges/project/ + blob_name = f'{BADGE_DIR}/{badge_name}' + + destination_blob_name = f'{DESTINATION_BADGE_DIR}/{project}.{extension}' + + status_bucket = get_storage_client().get_bucket(STATUS_BUCKET) + badge_blob = status_bucket.blob(blob_name) + status_bucket.copy_blob(badge_blob, + status_bucket, + new_name=destination_blob_name) + + +def upload_log(build_id): + """Upload log file to GCS.""" + status_bucket = get_storage_client().get_bucket(STATUS_BUCKET) + gcb_bucket = get_storage_client().get_bucket(build_project.GCB_LOGS_BUCKET) + log_name = f'log-{build_id}.txt' + log = gcb_bucket.blob(log_name) + dest_log = status_bucket.blob(log_name) + + if not log.exists(): + print('Failed to find build log', log_name, file=sys.stderr) + return False + + if dest_log.exists(): + return True + + gcb_bucket.copy_blob(log, status_bucket) + return True + + +# pylint: disable=no-member +# def update_status(event, context): +# """Entry point for cloud function to update build statuses and badges.""" +# del context + +# if 'data' in event: +# status_type = base64.b64decode(event['data']).decode() +# else: +# raise RuntimeError('No data') + +# if status_type == 'badges': +# update_badges() +# return + +# if status_type == 'fuzzing': +# tag = build_project.FUZZING_BUILD_TYPE +# status_filename = FUZZING_STATUS_FILENAME +# elif status_type == 'coverage': +# tag = build_and_run_coverage.COVERAGE_BUILD_TYPE +# status_filename = COVERAGE_STATUS_FILENAME +# else: +# raise RuntimeError('Invalid build status type ' + status_type) + +# with ndb.Client().context(): +# update_build_status(tag, status_filename) + + +def load_status_from_gcs(filename): + """Load statuses from bucket.""" + status_bucket = get_storage_client().get_bucket(STATUS_BUCKET) + status = json.loads(status_bucket.blob(filename).download_as_string()) + result = {} + + for project in status['projects']: + if project['history']: + result[project['name']] = project['history'][0]['success'] + + return result + + +def update_badges(): + """Update badges.""" + project_build_statuses = load_status_from_gcs(FUZZING_STATUS_FILENAME) + coverage_build_statuses = load_status_from_gcs(COVERAGE_STATUS_FILENAME) + + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as executor: + futures = [] + with ndb.Client().context(): + for project in Project.query(): + if project.name not in project_build_statuses: + continue + # Certain projects (e.g. JVM and Python) do not have any coverage + # builds, but should still receive a badge. + coverage_build_status = None + if project.name in coverage_build_statuses: + coverage_build_status = coverage_build_statuses[project.name] + + futures.append( + executor.submit(update_build_badges, project.name, + project_build_statuses[project.name], + coverage_build_status)) + concurrent.futures.wait(futures) + +if __name__ == "__main__": + #not sure on this! + update_badges() + + fuzz_tag = build_project.FUZZING_BUILD_TYPE + fuzz_status_filename = FUZZING_STATUS_FILENAME + + with ndb.Client().context(): + update_build_status(fuzz_tag, fuzz_status_filename) + + coverage_tag = build_and_run_coverage.COVERAGE_BUILD_TYPE + coverage_status_filename = COVERAGE_STATUS_FILENAME + + with ndb.Client().context(): + update_build_status(coverage_tag, coverage_status_filename) From 61211a55226380e0009a43a4fee932497aa81a4f Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Tue, 2 Nov 2021 16:29:38 -0700 Subject: [PATCH 02/23] fixed the yaml file path --- infra/build/build_status/cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/build/build_status/cloudbuild.yaml b/infra/build/build_status/cloudbuild.yaml index 21e554713914..803cdde3ed79 100644 --- a/infra/build/build_status/cloudbuild.yaml +++ b/infra/build/build_status/cloudbuild.yaml @@ -5,6 +5,6 @@ steps: - '-t' - gcr.io/oss-fuzz-base/build-status - '-f' - - infra/build_status/Dockerfile + - infra/build/build_status/Dockerfile - name: 'gcr.io/cloud-builders/docker' args: ['run', 'gcr.io/oss-fuzz-base/build-status'] \ No newline at end of file From 80a69cfb96c217bf2a1b56a3275eca263aa6b816 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Tue, 2 Nov 2021 16:38:47 -0700 Subject: [PATCH 03/23] fixed path to build --- infra/build/build_status/cloudbuild.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/infra/build/build_status/cloudbuild.yaml b/infra/build/build_status/cloudbuild.yaml index 803cdde3ed79..ec028a1a098a 100644 --- a/infra/build/build_status/cloudbuild.yaml +++ b/infra/build/build_status/cloudbuild.yaml @@ -6,5 +6,6 @@ steps: - gcr.io/oss-fuzz-base/build-status - '-f' - infra/build/build_status/Dockerfile + - ./build/ - name: 'gcr.io/cloud-builders/docker' args: ['run', 'gcr.io/oss-fuzz-base/build-status'] \ No newline at end of file From ed4de10cef0fa137ba476fe6391415c0eb3a9dd2 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Tue, 2 Nov 2021 16:40:17 -0700 Subject: [PATCH 04/23] fixed path to build to . --- infra/build/build_status/cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/build/build_status/cloudbuild.yaml b/infra/build/build_status/cloudbuild.yaml index ec028a1a098a..ce9ad0369cdb 100644 --- a/infra/build/build_status/cloudbuild.yaml +++ b/infra/build/build_status/cloudbuild.yaml @@ -6,6 +6,6 @@ steps: - gcr.io/oss-fuzz-base/build-status - '-f' - infra/build/build_status/Dockerfile - - ./build/ + - . - name: 'gcr.io/cloud-builders/docker' args: ['run', 'gcr.io/oss-fuzz-base/build-status'] \ No newline at end of file From 8182b6e411c38eedeabe8c6692efa021aaf68dfe Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Tue, 2 Nov 2021 17:40:55 -0700 Subject: [PATCH 05/23] fix OSS-Fuzz root in Dockerfile --- infra/build/build_status/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/build/build_status/Dockerfile b/infra/build/build_status/Dockerfile index 7d88c110adb8..a197e71e3dc2 100644 --- a/infra/build/build_status/Dockerfile +++ b/infra/build/build_status/Dockerfile @@ -2,7 +2,7 @@ FROM gcr.io/oss-fuzz-base/base-runner RUN mkdir -p /opt/oss-fuzz/infra/build_status -COPY ./functions/* /opt/oss-fuzz/infra/build_status/ -COPY ./build_status/* /opt/oss-fuzz/infra/build_status/ +COPY ${OSS_FUZZ_ROOT}/infra/functions/* /opt/oss-fuzz/infra/build_status/ +COPY ${OSS_FUZZ_ROOT}/infra/build_status/* /opt/oss-fuzz/infra/build_status/ RUN pip install -r /opt/oss-fuzz/infra/build_status/requirements.txt ENTRYPOINT [ "python3", "/opt/oss-fuzz/infra/build_status/update_build_status.py" ] \ No newline at end of file From 900d1d7a9eb97b9c1298cf52061fcf402faa0904 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Tue, 2 Nov 2021 17:50:36 -0700 Subject: [PATCH 06/23] fix OSS-Fuzz root in Dockerfile, added infra/build --- infra/build/build_status/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/build/build_status/Dockerfile b/infra/build/build_status/Dockerfile index a197e71e3dc2..abca44d5c119 100644 --- a/infra/build/build_status/Dockerfile +++ b/infra/build/build_status/Dockerfile @@ -2,7 +2,7 @@ FROM gcr.io/oss-fuzz-base/base-runner RUN mkdir -p /opt/oss-fuzz/infra/build_status -COPY ${OSS_FUZZ_ROOT}/infra/functions/* /opt/oss-fuzz/infra/build_status/ -COPY ${OSS_FUZZ_ROOT}/infra/build_status/* /opt/oss-fuzz/infra/build_status/ +COPY ${OSS_FUZZ_ROOT}/infra/build/functions/* /opt/oss-fuzz/infra/build_status/ +COPY ${OSS_FUZZ_ROOT}/infra/build/build_status/* /opt/oss-fuzz/infra/build_status/ RUN pip install -r /opt/oss-fuzz/infra/build_status/requirements.txt ENTRYPOINT [ "python3", "/opt/oss-fuzz/infra/build_status/update_build_status.py" ] \ No newline at end of file From 14272ba1159a9e8d76a25986f83766731e45bd0e Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Wed, 3 Nov 2021 13:46:22 -0700 Subject: [PATCH 07/23] fix image --- infra/build/build_status/cloudbuild.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/build/build_status/cloudbuild.yaml b/infra/build/build_status/cloudbuild.yaml index ce9ad0369cdb..c9eeece9782e 100644 --- a/infra/build/build_status/cloudbuild.yaml +++ b/infra/build/build_status/cloudbuild.yaml @@ -7,5 +7,5 @@ steps: - '-f' - infra/build/build_status/Dockerfile - . -- name: 'gcr.io/cloud-builders/docker' - args: ['run', 'gcr.io/oss-fuzz-base/build-status'] \ No newline at end of file +- name: 'gcr.io/oss-fuzz-base/build-status' + args: [] \ No newline at end of file From 9a33a35dca344686de47b7d96e46fb13bd36c1b7 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Wed, 3 Nov 2021 13:49:19 -0700 Subject: [PATCH 08/23] fix the order --- .../build/build_status/update_build_status.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/infra/build/build_status/update_build_status.py b/infra/build/build_status/update_build_status.py index a807c00d3b89..67a8542023ea 100644 --- a/infra/build/build_status/update_build_status.py +++ b/infra/build/build_status/update_build_status.py @@ -303,17 +303,17 @@ def update_badges(): concurrent.futures.wait(futures) if __name__ == "__main__": - #not sure on this! - update_badges() + #not sure on this! - fuzz_tag = build_project.FUZZING_BUILD_TYPE - fuzz_status_filename = FUZZING_STATUS_FILENAME + fuzz_tag = build_project.FUZZING_BUILD_TYPE + fuzz_status_filename = FUZZING_STATUS_FILENAME - with ndb.Client().context(): - update_build_status(fuzz_tag, fuzz_status_filename) + with ndb.Client().context(): + update_build_status(fuzz_tag, fuzz_status_filename) - coverage_tag = build_and_run_coverage.COVERAGE_BUILD_TYPE - coverage_status_filename = COVERAGE_STATUS_FILENAME + coverage_tag = build_and_run_coverage.COVERAGE_BUILD_TYPE + coverage_status_filename = COVERAGE_STATUS_FILENAME - with ndb.Client().context(): - update_build_status(coverage_tag, coverage_status_filename) + with ndb.Client().context(): + update_build_status(coverage_tag, coverage_status_filename) + update_badges() From d08f5f95bafc1724563999e8cb7732b0ed82a07a Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Wed, 3 Nov 2021 14:03:10 -0700 Subject: [PATCH 09/23] added logging option --- infra/build/build_status/cloudbuild.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/infra/build/build_status/cloudbuild.yaml b/infra/build/build_status/cloudbuild.yaml index c9eeece9782e..ac30cb981f30 100644 --- a/infra/build/build_status/cloudbuild.yaml +++ b/infra/build/build_status/cloudbuild.yaml @@ -8,4 +8,6 @@ steps: - infra/build/build_status/Dockerfile - . - name: 'gcr.io/oss-fuzz-base/build-status' - args: [] \ No newline at end of file + args: [] +options: + logging: CLOUD_LOGGING_ONLY \ No newline at end of file From 88f2ade5f9ac5ded7c35d6b158b5c8ff333a8592 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Wed, 3 Nov 2021 15:30:50 -0700 Subject: [PATCH 10/23] None logging option --- infra/build/build_status/cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/build/build_status/cloudbuild.yaml b/infra/build/build_status/cloudbuild.yaml index ac30cb981f30..1f8f13da5844 100644 --- a/infra/build/build_status/cloudbuild.yaml +++ b/infra/build/build_status/cloudbuild.yaml @@ -10,4 +10,4 @@ steps: - name: 'gcr.io/oss-fuzz-base/build-status' args: [] options: - logging: CLOUD_LOGGING_ONLY \ No newline at end of file + logging: None \ No newline at end of file From 7a077497ee4fa0b65ed063c75f749eb082c6567f Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Wed, 3 Nov 2021 15:31:56 -0700 Subject: [PATCH 11/23] fix logging option --- infra/build/build_status/cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/build/build_status/cloudbuild.yaml b/infra/build/build_status/cloudbuild.yaml index 1f8f13da5844..ac30cb981f30 100644 --- a/infra/build/build_status/cloudbuild.yaml +++ b/infra/build/build_status/cloudbuild.yaml @@ -10,4 +10,4 @@ steps: - name: 'gcr.io/oss-fuzz-base/build-status' args: [] options: - logging: None \ No newline at end of file + logging: CLOUD_LOGGING_ONLY \ No newline at end of file From 1ef658a303425cdcb1eb8bc9722ab197a60fd217 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Wed, 3 Nov 2021 19:50:13 -0700 Subject: [PATCH 12/23] set cloudbuild timeout --- infra/build/build_status/cloudbuild.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/infra/build/build_status/cloudbuild.yaml b/infra/build/build_status/cloudbuild.yaml index ac30cb981f30..abf1ec602d7f 100644 --- a/infra/build/build_status/cloudbuild.yaml +++ b/infra/build/build_status/cloudbuild.yaml @@ -9,5 +9,6 @@ steps: - . - name: 'gcr.io/oss-fuzz-base/build-status' args: [] + timeout: 3600s options: logging: CLOUD_LOGGING_ONLY \ No newline at end of file From 3f7ace4daee474f5c5cdb622fff456c631be29d6 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Wed, 3 Nov 2021 20:01:56 -0700 Subject: [PATCH 13/23] set total cloudbuild timeout --- infra/build/build_status/cloudbuild.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infra/build/build_status/cloudbuild.yaml b/infra/build/build_status/cloudbuild.yaml index abf1ec602d7f..c3cf6b70de17 100644 --- a/infra/build/build_status/cloudbuild.yaml +++ b/infra/build/build_status/cloudbuild.yaml @@ -11,4 +11,5 @@ steps: args: [] timeout: 3600s options: - logging: CLOUD_LOGGING_ONLY \ No newline at end of file + logging: CLOUD_LOGGING_ONLY +timeout: 3600s \ No newline at end of file From 68dea685328a5c6b87cb550fb380d9c2c7ec3e65 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Thu, 4 Nov 2021 09:35:07 -0700 Subject: [PATCH 14/23] clean-up and adding copyright box --- infra/build/build_status/Dockerfile | 18 +++++++++++- .../build/build_status/update_build_status.py | 29 ------------------- 2 files changed, 17 insertions(+), 30 deletions(-) diff --git a/infra/build/build_status/Dockerfile b/infra/build/build_status/Dockerfile index abca44d5c119..34773765a189 100644 --- a/infra/build/build_status/Dockerfile +++ b/infra/build/build_status/Dockerfile @@ -1,8 +1,24 @@ -FROM gcr.io/oss-fuzz-base/base-runner +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +FROM gcr.io/oss-fuzz-base/base-runner RUN mkdir -p /opt/oss-fuzz/infra/build_status COPY ${OSS_FUZZ_ROOT}/infra/build/functions/* /opt/oss-fuzz/infra/build_status/ COPY ${OSS_FUZZ_ROOT}/infra/build/build_status/* /opt/oss-fuzz/infra/build_status/ RUN pip install -r /opt/oss-fuzz/infra/build_status/requirements.txt + ENTRYPOINT [ "python3", "/opt/oss-fuzz/infra/build_status/update_build_status.py" ] \ No newline at end of file diff --git a/infra/build/build_status/update_build_status.py b/infra/build/build_status/update_build_status.py index 67a8542023ea..b46b258e9d49 100644 --- a/infra/build/build_status/update_build_status.py +++ b/infra/build/build_status/update_build_status.py @@ -239,33 +239,6 @@ def upload_log(build_id): return True -# pylint: disable=no-member -# def update_status(event, context): -# """Entry point for cloud function to update build statuses and badges.""" -# del context - -# if 'data' in event: -# status_type = base64.b64decode(event['data']).decode() -# else: -# raise RuntimeError('No data') - -# if status_type == 'badges': -# update_badges() -# return - -# if status_type == 'fuzzing': -# tag = build_project.FUZZING_BUILD_TYPE -# status_filename = FUZZING_STATUS_FILENAME -# elif status_type == 'coverage': -# tag = build_and_run_coverage.COVERAGE_BUILD_TYPE -# status_filename = COVERAGE_STATUS_FILENAME -# else: -# raise RuntimeError('Invalid build status type ' + status_type) - -# with ndb.Client().context(): -# update_build_status(tag, status_filename) - - def load_status_from_gcs(filename): """Load statuses from bucket.""" status_bucket = get_storage_client().get_bucket(STATUS_BUCKET) @@ -303,8 +276,6 @@ def update_badges(): concurrent.futures.wait(futures) if __name__ == "__main__": - #not sure on this! - fuzz_tag = build_project.FUZZING_BUILD_TYPE fuzz_status_filename = FUZZING_STATUS_FILENAME From 8e49e568bd64184e3c6b7a9afe04bdc52ce76023 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Thu, 4 Nov 2021 10:30:54 -0700 Subject: [PATCH 15/23] fix lint issues --- infra/build/build_status/update_build_status.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/infra/build/build_status/update_build_status.py b/infra/build/build_status/update_build_status.py index b46b258e9d49..5b5f010ec1be 100644 --- a/infra/build/build_status/update_build_status.py +++ b/infra/build/build_status/update_build_status.py @@ -14,7 +14,6 @@ # ################################################################################ """Cloud function to request builds.""" -import base64 import concurrent.futures import json import sys @@ -280,11 +279,11 @@ def update_badges(): fuzz_status_filename = FUZZING_STATUS_FILENAME with ndb.Client().context(): - update_build_status(fuzz_tag, fuzz_status_filename) + update_build_status(fuzz_tag, fuzz_status_filename) coverage_tag = build_and_run_coverage.COVERAGE_BUILD_TYPE coverage_status_filename = COVERAGE_STATUS_FILENAME with ndb.Client().context(): - update_build_status(coverage_tag, coverage_status_filename) + update_build_status(coverage_tag, coverage_status_filename) update_badges() From 9ea34846f5f63b381969afaeaf9c4cc818fbac98 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Thu, 4 Nov 2021 10:39:08 -0700 Subject: [PATCH 16/23] fix lint main issue --- infra/build/build_status/update_build_status.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/infra/build/build_status/update_build_status.py b/infra/build/build_status/update_build_status.py index 5b5f010ec1be..713b25bfd524 100644 --- a/infra/build/build_status/update_build_status.py +++ b/infra/build/build_status/update_build_status.py @@ -274,7 +274,7 @@ def update_badges(): coverage_build_status)) concurrent.futures.wait(futures) -if __name__ == "__main__": +def main(): fuzz_tag = build_project.FUZZING_BUILD_TYPE fuzz_status_filename = FUZZING_STATUS_FILENAME @@ -287,3 +287,6 @@ def update_badges(): with ndb.Client().context(): update_build_status(coverage_tag, coverage_status_filename) update_badges() + +if __name__ == "__main__": + main() \ No newline at end of file From 67148c950eed0a349072ef8268b8719531235894 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Thu, 4 Nov 2021 10:48:13 -0700 Subject: [PATCH 17/23] fix more lint issues --- infra/build/build_status/update_build_status.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infra/build/build_status/update_build_status.py b/infra/build/build_status/update_build_status.py index 713b25bfd524..4d5a342c02db 100644 --- a/infra/build/build_status/update_build_status.py +++ b/infra/build/build_status/update_build_status.py @@ -275,6 +275,7 @@ def update_badges(): concurrent.futures.wait(futures) def main(): + """Entry point for cloudbuild""" fuzz_tag = build_project.FUZZING_BUILD_TYPE fuzz_status_filename = FUZZING_STATUS_FILENAME @@ -289,4 +290,4 @@ def main(): update_badges() if __name__ == "__main__": - main() \ No newline at end of file + main() From 9240c55a55d96e8c2c76953b93b681ea2309ffa2 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Thu, 4 Nov 2021 13:56:25 -0700 Subject: [PATCH 18/23] fix lint issues using presubmit --- infra/build/build_status/update_build_status.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infra/build/build_status/update_build_status.py b/infra/build/build_status/update_build_status.py index 4d5a342c02db..ebf3cb11867b 100644 --- a/infra/build/build_status/update_build_status.py +++ b/infra/build/build_status/update_build_status.py @@ -274,6 +274,7 @@ def update_badges(): coverage_build_status)) concurrent.futures.wait(futures) + def main(): """Entry point for cloudbuild""" fuzz_tag = build_project.FUZZING_BUILD_TYPE @@ -289,5 +290,6 @@ def main(): update_build_status(coverage_tag, coverage_status_filename) update_badges() + if __name__ == "__main__": main() From 560f5274a2bfee109836cd6f475e2a13fe15a7bb Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Thu, 4 Nov 2021 16:23:03 -0700 Subject: [PATCH 19/23] fix copyright year --- infra/build/build_status/Dockerfile | 2 +- infra/build/build_status/update_build_status.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/build/build_status/Dockerfile b/infra/build/build_status/Dockerfile index 34773765a189..4023eafa2c7c 100644 --- a/infra/build/build_status/Dockerfile +++ b/infra/build/build_status/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2020 Google Inc. +# Copyright 2021 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/infra/build/build_status/update_build_status.py b/infra/build/build_status/update_build_status.py index ebf3cb11867b..44fcc1ac6521 100644 --- a/infra/build/build_status/update_build_status.py +++ b/infra/build/build_status/update_build_status.py @@ -1,4 +1,4 @@ -# Copyright 2020 Google Inc. +# Copyright 2021 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -291,5 +291,5 @@ def main(): update_badges() -if __name__ == "__main__": +if __name__ == '__main__': main() From 4a4942a79b0c5f668a40ee91247cd3941826774a Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Thu, 11 Nov 2021 09:18:41 -0800 Subject: [PATCH 20/23] clean up main, and remove old update_build_status.py --- .../build/build_status/update_build_status.py | 45 ++- infra/build/functions/update_build_status.py | 303 ------------------ 2 files changed, 21 insertions(+), 327 deletions(-) delete mode 100644 infra/build/functions/update_build_status.py diff --git a/infra/build/build_status/update_build_status.py b/infra/build/build_status/update_build_status.py index 44fcc1ac6521..f1797d81c25a 100644 --- a/infra/build/build_status/update_build_status.py +++ b/infra/build/build_status/update_build_status.py @@ -258,38 +258,35 @@ def update_badges(): with concurrent.futures.ThreadPoolExecutor(max_workers=32) as executor: futures = [] - with ndb.Client().context(): - for project in Project.query(): - if project.name not in project_build_statuses: - continue - # Certain projects (e.g. JVM and Python) do not have any coverage - # builds, but should still receive a badge. - coverage_build_status = None - if project.name in coverage_build_statuses: - coverage_build_status = coverage_build_statuses[project.name] - - futures.append( - executor.submit(update_build_badges, project.name, - project_build_statuses[project.name], - coverage_build_status)) + for project in Project.query(): + if project.name not in project_build_statuses: + continue + # Certain projects (e.g. JVM and Python) do not have any coverage + # builds, but should still receive a badge. + coverage_build_status = None + if project.name in coverage_build_statuses: + coverage_build_status = coverage_build_statuses[project.name] + + futures.append( + executor.submit(update_build_badges, project.name, + project_build_statuses[project.name], + coverage_build_status)) concurrent.futures.wait(futures) def main(): """Entry point for cloudbuild""" - fuzz_tag = build_project.FUZZING_BUILD_TYPE - fuzz_status_filename = FUZZING_STATUS_FILENAME + config = ( + (build_project.FUZZING_BUILD_TYPE, FUZZING_STATUS_FILENAME), + (build_and_run_coverage.COVERAGE_BUILD_TYPE, COVERAGE_STATUS_FILENAME) + ) - with ndb.Client().context(): - update_build_status(fuzz_tag, fuzz_status_filename) + for tag, filename in config: + update_build_status(tag, filename) - coverage_tag = build_and_run_coverage.COVERAGE_BUILD_TYPE - coverage_status_filename = COVERAGE_STATUS_FILENAME - - with ndb.Client().context(): - update_build_status(coverage_tag, coverage_status_filename) update_badges() if __name__ == '__main__': - main() + with ndb.Client.context(): + main() diff --git a/infra/build/functions/update_build_status.py b/infra/build/functions/update_build_status.py deleted file mode 100644 index 92721662835a..000000000000 --- a/infra/build/functions/update_build_status.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright 2020 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -"""Cloud function to request builds.""" -import base64 -import concurrent.futures -import json -import sys - -import google.auth -from googleapiclient.discovery import build -from google.cloud import ndb -from google.cloud import storage - -import build_and_run_coverage -import build_project -from datastore_entities import BuildsHistory -from datastore_entities import LastSuccessfulBuild -from datastore_entities import Project - -BADGE_DIR = 'badge_images' -BADGE_IMAGE_TYPES = {'svg': 'image/svg+xml', 'png': 'image/png'} -DESTINATION_BADGE_DIR = 'badges' -MAX_BUILD_LOGS = 7 - -STATUS_BUCKET = 'oss-fuzz-build-logs' - -FUZZING_STATUS_FILENAME = 'status.json' -COVERAGE_STATUS_FILENAME = 'status-coverage.json' - -# pylint: disable=invalid-name -_client = None - - -class MissingBuildLogError(Exception): - """Missing build log file in cloud storage.""" - - -# pylint: disable=global-statement -def get_storage_client(): - """Return storage client.""" - global _client - if not _client: - _client = storage.Client() - - return _client - - -def is_build_successful(build_obj): - """Check build success.""" - return build_obj['status'] == 'SUCCESS' - - -def upload_status(data, status_filename): - """Upload json file to cloud storage.""" - bucket = get_storage_client().get_bucket(STATUS_BUCKET) - blob = bucket.blob(status_filename) - blob.cache_control = 'no-cache' - blob.upload_from_string(json.dumps(data), content_type='application/json') - - -def sort_projects(projects): - """Sort projects in order Failures, Successes, Not yet built.""" - - def key_func(project): - if not project['history']: - return 2 # Order projects without history last. - - if project['history'][0]['success']: - # Successful builds come second. - return 1 - - # Build failures come first. - return 0 - - projects.sort(key=key_func) - - -def get_build(cloudbuild, image_project, build_id): - """Get build object from cloudbuild.""" - return cloudbuild.projects().builds().get(projectId=image_project, - id=build_id).execute() - - -def update_last_successful_build(project, build_tag): - """Update last successful build.""" - last_successful_build = ndb.Key(LastSuccessfulBuild, - project['name'] + '-' + build_tag).get() - if not last_successful_build and 'last_successful_build' not in project: - return - - if 'last_successful_build' not in project: - project['last_successful_build'] = { - 'build_id': last_successful_build.build_id, - 'finish_time': last_successful_build.finish_time - } - else: - if last_successful_build: - last_successful_build.build_id = project['last_successful_build'][ - 'build_id'] - last_successful_build.finish_time = project['last_successful_build'][ - 'finish_time'] - else: - last_successful_build = LastSuccessfulBuild( - id=project['name'] + '-' + build_tag, - project=project['name'], - build_id=project['last_successful_build']['build_id'], - finish_time=project['last_successful_build']['finish_time']) - last_successful_build.put() - - -# pylint: disable=no-member -def get_build_history(build_ids): - """Returns build object for the last finished build of project.""" - credentials, image_project = google.auth.default() - cloudbuild = build('cloudbuild', - 'v1', - credentials=credentials, - cache_discovery=False) - - history = [] - last_successful_build = None - - for build_id in reversed(build_ids): - project_build = get_build(cloudbuild, image_project, build_id) - if project_build['status'] not in ('SUCCESS', 'FAILURE', 'TIMEOUT'): - continue - - if (not last_successful_build and is_build_successful(project_build)): - last_successful_build = { - 'build_id': build_id, - 'finish_time': project_build['finishTime'], - } - - if not upload_log(build_id): - log_name = f'log-{build_id}' - raise MissingBuildLogError(f'Missing build log file {log_name}') - - history.append({ - 'build_id': build_id, - 'finish_time': project_build['finishTime'], - 'success': is_build_successful(project_build) - }) - - if len(history) == MAX_BUILD_LOGS: - break - - project = {'history': history} - if last_successful_build: - project['last_successful_build'] = last_successful_build - return project - - -# pylint: disable=too-many-locals -def update_build_status(build_tag, status_filename): - """Update build statuses.""" - projects = [] - - def process_project(project_build): - """Process a project.""" - project = get_build_history(project_build.build_ids) - project['name'] = project_build.project - print('Processing project', project['name']) - return project - - with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: - futures = [] - for project_build in BuildsHistory.query( - BuildsHistory.build_tag == build_tag).order('project'): - futures.append(executor.submit(process_project, project_build)) - - for future in concurrent.futures.as_completed(futures): - project = future.result() - update_last_successful_build(project, build_tag) - projects.append(project) - - sort_projects(projects) - data = {'projects': projects} - upload_status(data, status_filename) - - -def update_build_badges(project, last_build_successful, - last_coverage_build_successful): - """Upload badges of given project.""" - badge = 'building' - # last_coverage_build_successful is False if there was an unsuccessful build - # and None if the target does not support coverage (e.g. Python or Java - # targets). - if last_coverage_build_successful is False: - badge = 'coverage_failing' - if not last_build_successful: - badge = 'failing' - - print(f'[badge] {project}: {badge}') - - for extension in BADGE_IMAGE_TYPES: - badge_name = f'{badge}.{extension}' - - # Copy blob from badge_images/badge_name to badges/project/ - blob_name = f'{BADGE_DIR}/{badge_name}' - - destination_blob_name = f'{DESTINATION_BADGE_DIR}/{project}.{extension}' - - status_bucket = get_storage_client().get_bucket(STATUS_BUCKET) - badge_blob = status_bucket.blob(blob_name) - status_bucket.copy_blob(badge_blob, - status_bucket, - new_name=destination_blob_name) - - -def upload_log(build_id): - """Upload log file to GCS.""" - status_bucket = get_storage_client().get_bucket(STATUS_BUCKET) - gcb_bucket = get_storage_client().get_bucket(build_project.GCB_LOGS_BUCKET) - log_name = f'log-{build_id}.txt' - log = gcb_bucket.blob(log_name) - dest_log = status_bucket.blob(log_name) - - if not log.exists(): - print('Failed to find build log', log_name, file=sys.stderr) - return False - - if dest_log.exists(): - return True - - gcb_bucket.copy_blob(log, status_bucket) - return True - - -# pylint: disable=no-member -def update_status(event, context): - """Entry point for cloud function to update build statuses and badges.""" - del context - - if 'data' in event: - status_type = base64.b64decode(event['data']).decode() - else: - raise RuntimeError('No data') - - if status_type == 'badges': - update_badges() - return - - if status_type == 'fuzzing': - tag = build_project.FUZZING_BUILD_TYPE - status_filename = FUZZING_STATUS_FILENAME - elif status_type == 'coverage': - tag = build_and_run_coverage.COVERAGE_BUILD_TYPE - status_filename = COVERAGE_STATUS_FILENAME - else: - raise RuntimeError('Invalid build status type ' + status_type) - - with ndb.Client().context(): - update_build_status(tag, status_filename) - - -def load_status_from_gcs(filename): - """Load statuses from bucket.""" - status_bucket = get_storage_client().get_bucket(STATUS_BUCKET) - status = json.loads(status_bucket.blob(filename).download_as_string()) - result = {} - - for project in status['projects']: - if project['history']: - result[project['name']] = project['history'][0]['success'] - - return result - - -def update_badges(): - """Update badges.""" - project_build_statuses = load_status_from_gcs(FUZZING_STATUS_FILENAME) - coverage_build_statuses = load_status_from_gcs(COVERAGE_STATUS_FILENAME) - - with concurrent.futures.ThreadPoolExecutor(max_workers=32) as executor: - futures = [] - with ndb.Client().context(): - for project in Project.query(): - if project.name not in project_build_statuses: - continue - # Certain projects (e.g. JVM and Python) do not have any coverage - # builds, but should still receive a badge. - coverage_build_status = None - if project.name in coverage_build_statuses: - coverage_build_status = coverage_build_statuses[project.name] - - futures.append( - executor.submit(update_build_badges, project.name, - project_build_statuses[project.name], - coverage_build_status)) - concurrent.futures.wait(futures) From f57a3c0c78aa55bb672e1390a90eee5e5c54629d Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Thu, 11 Nov 2021 15:21:32 -0800 Subject: [PATCH 21/23] fix call to ndb.Client() --- infra/build/build_status/update_build_status.py | 9 ++++----- .../update_build_status_test.py | 0 2 files changed, 4 insertions(+), 5 deletions(-) rename infra/build/{functions => build_status}/update_build_status_test.py (100%) diff --git a/infra/build/build_status/update_build_status.py b/infra/build/build_status/update_build_status.py index f1797d81c25a..68d793a6680e 100644 --- a/infra/build/build_status/update_build_status.py +++ b/infra/build/build_status/update_build_status.py @@ -276,10 +276,9 @@ def update_badges(): def main(): """Entry point for cloudbuild""" - config = ( - (build_project.FUZZING_BUILD_TYPE, FUZZING_STATUS_FILENAME), - (build_and_run_coverage.COVERAGE_BUILD_TYPE, COVERAGE_STATUS_FILENAME) - ) + config = ((build_project.FUZZING_BUILD_TYPE, FUZZING_STATUS_FILENAME), + (build_and_run_coverage.COVERAGE_BUILD_TYPE, + COVERAGE_STATUS_FILENAME)) for tag, filename in config: update_build_status(tag, filename) @@ -288,5 +287,5 @@ def main(): if __name__ == '__main__': - with ndb.Client.context(): + with ndb.Client().context(): main() diff --git a/infra/build/functions/update_build_status_test.py b/infra/build/build_status/update_build_status_test.py similarity index 100% rename from infra/build/functions/update_build_status_test.py rename to infra/build/build_status/update_build_status_test.py From 10c4309dd2d13371e8dd74097c0bb108425df68b Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Tue, 16 Nov 2021 13:57:56 -0800 Subject: [PATCH 22/23] fixing final nits --- infra/build/build_status/Dockerfile | 2 +- .../build/build_status/update_build_status.py | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/infra/build/build_status/Dockerfile b/infra/build/build_status/Dockerfile index 4023eafa2c7c..e05f7fc93895 100644 --- a/infra/build/build_status/Dockerfile +++ b/infra/build/build_status/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2021 Google Inc. +# Copyright 2021 Google LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/infra/build/build_status/update_build_status.py b/infra/build/build_status/update_build_status.py index 68d793a6680e..7bc6073071d8 100644 --- a/infra/build/build_status/update_build_status.py +++ b/infra/build/build_status/update_build_status.py @@ -1,4 +1,4 @@ -# Copyright 2021 Google Inc. +# Copyright 2020 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -34,7 +34,7 @@ DESTINATION_BADGE_DIR = 'badges' MAX_BUILD_LOGS = 7 -STATUS_BUCKET = 'oss-fuzz-build-logs-test' +STATUS_BUCKET = 'oss-fuzz-build-logs' FUZZING_STATUS_FILENAME = 'status.json' COVERAGE_STATUS_FILENAME = 'status-coverage.json' @@ -276,16 +276,16 @@ def update_badges(): def main(): """Entry point for cloudbuild""" - config = ((build_project.FUZZING_BUILD_TYPE, FUZZING_STATUS_FILENAME), - (build_and_run_coverage.COVERAGE_BUILD_TYPE, - COVERAGE_STATUS_FILENAME)) + with ndb.Client().context(): + configs = ((build_project.FUZZING_BUILD_TYPE, FUZZING_STATUS_FILENAME), + (build_and_run_coverage.COVERAGE_BUILD_TYPE, + COVERAGE_STATUS_FILENAME)) - for tag, filename in config: - update_build_status(tag, filename) + for tag, filename in configs: + update_build_status(tag, filename) - update_badges() + update_badges() if __name__ == '__main__': - with ndb.Client().context(): - main() + main() From d224f7973e3e209c99f207d34c62eb1035633615 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Tue, 21 Dec 2021 14:38:55 -0800 Subject: [PATCH 23/23] Added steps to the coverage build to integrate Fuzz Introspector --- .../build/functions/build_and_run_coverage.py | 142 +++++++++++++++++- infra/build/functions/build_project.py | 9 +- 2 files changed, 145 insertions(+), 6 deletions(-) diff --git a/infra/build/functions/build_and_run_coverage.py b/infra/build/functions/build_and_run_coverage.py index 1195776d96e8..578d8d6ae90b 100755 --- a/infra/build/functions/build_and_run_coverage.py +++ b/infra/build/functions/build_and_run_coverage.py @@ -36,6 +36,7 @@ # Where code coverage reports need to be uploaded to. COVERAGE_BUCKET_NAME = 'oss-fuzz-coverage' +INTROSPECTOR_BUCKET_NAME = 'oss-fuzz-introspector' # This is needed for ClusterFuzz to pick up the most recent reports data. @@ -67,6 +68,28 @@ def get_upload_url(self, upload_type): f'/{upload_type}/{self.date}') +class IntrospectorBucket: # pylint: disable=too-few-public-methods + """Class representing the fuzz introspector GCS bucket.""" + + def __init__(self, project, date, platform, testing): + self.introspector_bucket_name = 'oss-fuzz-introspector' + if testing: + self.introspector_bucket_name += '-testing' + + self.date = date + self.project = project + self.html_report_url = ( + f'{build_lib.GCS_URL_BASENAME}{self.introspector_bucket_name}/{project}' + f'/reports/{date}/{platform}/index.html') + self.latest_report_info_url = (f'/{INTROSPECTOR_BUCKET_NAME}' + f'/latest_report_info/{project}.json') + + def get_upload_url(self, upload_type): + """Returns an upload url for |upload_type|.""" + return (f'gs://{self.introspector_bucket_name}/{self.project}' + f'/{upload_type}/{self.date}') + + def get_build_steps( # pylint: disable=too-many-locals, too-many-arguments project_name, project_yaml_contents, dockerfile_lines, image_project, base_images_project, config): @@ -121,11 +144,11 @@ def get_build_steps( # pylint: disable=too-many-locals, too-many-arguments coverage_env.append('FULL_SUMMARY_PER_TARGET=1') build_steps.append({ - 'name': - build_project.get_runner_image_name(base_images_project, - config.test_image_suffix), - 'env': - coverage_env, + 'name': # build_project.get_runner_image_name(base_images_project, + # config.test_image_suffix), + # 'gcr.io/oss-fuzz-base/base-runner-customized', #TODO: hacky! to be fixed + 'gcr.io/oss-fuzz-base/base-runner:introspector', + 'env': coverage_env, 'args': [ 'bash', '-c', ('for f in /corpus/*.zip; do unzip -q $f -d ${f%%.*} || (' @@ -223,6 +246,115 @@ def get_build_steps( # pylint: disable=too-many-locals, too-many-arguments build_lib.http_upload_step(latest_report_info_body, latest_report_info_url, LATEST_REPORT_INFO_CONTENT_TYPE)) + #removes index.html from the end of url + coverage_url = bucket.html_report_url[:-11] + build_steps.extend( + get_fuzz_introspector_steps(project, project_name, base_images_project, + config, coverage_url)) + return build_steps + + +def get_fuzz_introspector_steps(project, project_name, base_images_project, + config, coverage_url): + build_steps = [] + FI_dir = '/workspace/fuzz-introspector/' + oss_integration_dir = 'oss_fuzz_integration/' + + report_date = build_project.get_datetime_now().strftime('%Y%m%d') + bucket = IntrospectorBucket(project.name, report_date, PLATFORM, + config.testing) + + build = build_project.Build('libfuzzer', 'instrumentor', 'x86_64') + env = build_project.get_env(project.fuzzing_language, build) + + clone_step = { + 'args': [ + 'clone', 'https://github.com/ossf/fuzz-introspector.git', '--depth', + '1' + ], + 'name': 'gcr.io/cloud-builders/git', + } + build_steps.append(clone_step) + + build_steps.append({ + 'name': + build_project.get_runner_image_name(base_images_project, + config.test_image_suffix), + 'args': [ + 'bash', '-c', + (f'cd {FI_dir} && cd {oss_integration_dir}' + ' && sed -i \'s/\.\/infra\/base\-images\/all.sh/#\.\/infra\/base\-images\/all.sh/\'' + ' build_patched_oss_fuzz.sh' + ' && cat build_patched_oss_fuzz.sh' + ' && ./build_patched_oss_fuzz.sh') + ] + }) + + build_steps.append({ + 'name': + build_project.get_runner_image_name(base_images_project, + config.test_image_suffix), + 'args': [ + 'bash', '-c', + ('sed -i s/base-builder/base-builder:introspector/g ' + f'{FI_dir}{oss_integration_dir}oss-fuzz/projects/{project_name}/Dockerfile' + f' && cat {FI_dir}{oss_integration_dir}oss-fuzz/projects/{project_name}/Dockerfile' + ) + ] + }) + + build_steps.append({ + 'name': + 'gcr.io/cloud-builders/docker', + 'args': [ + 'build', + '-t', + f'gcr.io/oss-fuzz/{project_name}', + '--file', + f'{FI_dir}{oss_integration_dir}oss-fuzz/projects/{project_name}/Dockerfile', + f'{FI_dir}{oss_integration_dir}oss-fuzz/projects/{project_name}', + ], + }) + + build_steps.append( + build_project.get_compile_step(project, build, env, config.parallel)) + + #adjust coverage url + cov_url_escaped = coverage_url.replace("/", "\/").replace(":", "\:") + set_cov_url = ( + f'sed -i \'s/http\:\/\/localhost\:8008\/covreport\/linux/{cov_url_escaped}/\'' + ' /src/post-processing/main.py && cat /src/post-processing/main.py && ') + last_build_step = build_steps[-1] + last_args = last_build_step['args'] + last_bash_cmd = last_args[2] + last_bash_cmd = set_cov_url + last_bash_cmd + build_steps[-1]['args'][2] = last_bash_cmd + + build_steps.append({ + 'name': + build_project.get_runner_image_name(base_images_project, + config.test_image_suffix), + 'args': ['bash', '-c', ('du -a /workspace')] + }) + + # Upload the report. + upload_report_url = bucket.get_upload_url('inspector-report') + + # Delete the existing report as gsutil cannot overwrite it in a useful way due + # to the lack of `-T` option (it creates a subdir in the destination dir). + build_steps.append(build_lib.gsutil_rm_rf_step(upload_report_url)) + build_steps.append({ + 'name': + 'gcr.io/cloud-builders/gsutil', + 'args': [ + '-m', + 'cp', + '-r', + os.path.join(build.out, 'inspector-tmp'), + upload_report_url, + ], + }) + return build_steps diff --git a/infra/build/functions/build_project.py b/infra/build/functions/build_project.py index bdc7985e172f..12229de78cbd 100755 --- a/infra/build/functions/build_project.py +++ b/infra/build/functions/build_project.py @@ -195,12 +195,19 @@ def get_env(fuzzing_language, build): 'HOME': '/root', 'OUT': build.out, } + return list(sorted([f'{key}={value}' for key, value in env_dict.items()])) def get_compile_step(project, build, env, parallel): """Returns the GCB step for compiling |projects| fuzzers using |env|. The type of build is specified by |build|.""" + set_git_repo_env = '' #do nothing + if build.sanitizer == 'instrumentor': + set_git_repo_env = ( + ' && export GITHUB_REPO=$(grep -P -o "\S+github.com\S+" /workspace/oss-fuzz/projects/' + f'{project.name}/Dockerfile | xargs -L 1 | sed -e "s/.git$//") && set |grep GITHUB_REPO ' + ) failure_msg = ( '*' * 80 + '\nFailed to build.\nTo reproduce, run:\n' f'python infra/helper.py build_image {project.name}\n' @@ -218,7 +225,7 @@ def get_compile_step(project, build, env, parallel): # Dockerfile). Container Builder overrides our workdir so we need # to add this step to set it back. (f'rm -r /out && cd /src && cd {project.workdir} && ' - f'mkdir -p {build.out} && compile || ' + f'mkdir -p {build.out} {set_git_repo_env} && compile || ' f'(echo "{failure_msg}" && false)'), ], 'id': get_id('compile', build),