diff --git a/infra/build/build_status/Dockerfile b/infra/build/build_status/Dockerfile new file mode 100644 index 000000000000..e05f7fc93895 --- /dev/null +++ b/infra/build/build_status/Dockerfile @@ -0,0 +1,24 @@ +# Copyright 2021 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +FROM gcr.io/oss-fuzz-base/base-runner + +RUN mkdir -p /opt/oss-fuzz/infra/build_status +COPY ${OSS_FUZZ_ROOT}/infra/build/functions/* /opt/oss-fuzz/infra/build_status/ +COPY ${OSS_FUZZ_ROOT}/infra/build/build_status/* /opt/oss-fuzz/infra/build_status/ +RUN pip install -r /opt/oss-fuzz/infra/build_status/requirements.txt + +ENTRYPOINT [ "python3", "/opt/oss-fuzz/infra/build_status/update_build_status.py" ] \ No newline at end of file diff --git a/infra/build/build_status/cloudbuild.yaml b/infra/build/build_status/cloudbuild.yaml new file mode 100644 index 000000000000..c3cf6b70de17 --- /dev/null +++ b/infra/build/build_status/cloudbuild.yaml @@ -0,0 +1,15 @@ +steps: +- name: 'gcr.io/cloud-builders/docker' + args: + - build + - '-t' + - gcr.io/oss-fuzz-base/build-status + - '-f' + - infra/build/build_status/Dockerfile + - . +- name: 'gcr.io/oss-fuzz-base/build-status' + args: [] + timeout: 3600s +options: + logging: CLOUD_LOGGING_ONLY +timeout: 3600s \ No newline at end of file diff --git a/infra/build/functions/update_build_status.py b/infra/build/build_status/update_build_status.py similarity index 85% rename from infra/build/functions/update_build_status.py rename to infra/build/build_status/update_build_status.py index 92721662835a..7bc6073071d8 100644 --- a/infra/build/functions/update_build_status.py +++ b/infra/build/build_status/update_build_status.py @@ -14,7 +14,6 @@ # ################################################################################ """Cloud function to request builds.""" -import base64 import concurrent.futures import json import sys @@ -239,33 +238,6 @@ def upload_log(build_id): return True -# pylint: disable=no-member -def update_status(event, context): - """Entry point for cloud function to update build statuses and badges.""" - del context - - if 'data' in event: - status_type = base64.b64decode(event['data']).decode() - else: - raise RuntimeError('No data') - - if status_type == 'badges': - update_badges() - return - - if status_type == 'fuzzing': - tag = build_project.FUZZING_BUILD_TYPE - status_filename = FUZZING_STATUS_FILENAME - elif status_type == 'coverage': - tag = build_and_run_coverage.COVERAGE_BUILD_TYPE - status_filename = COVERAGE_STATUS_FILENAME - else: - raise RuntimeError('Invalid build status type ' + status_type) - - with ndb.Client().context(): - update_build_status(tag, status_filename) - - def load_status_from_gcs(filename): """Load statuses from bucket.""" status_bucket = get_storage_client().get_bucket(STATUS_BUCKET) @@ -286,18 +258,34 @@ def update_badges(): with concurrent.futures.ThreadPoolExecutor(max_workers=32) as executor: futures = [] - with ndb.Client().context(): - for project in Project.query(): - if project.name not in project_build_statuses: - continue - # Certain projects (e.g. JVM and Python) do not have any coverage - # builds, but should still receive a badge. - coverage_build_status = None - if project.name in coverage_build_statuses: - coverage_build_status = coverage_build_statuses[project.name] - - futures.append( - executor.submit(update_build_badges, project.name, - project_build_statuses[project.name], - coverage_build_status)) + for project in Project.query(): + if project.name not in project_build_statuses: + continue + # Certain projects (e.g. JVM and Python) do not have any coverage + # builds, but should still receive a badge. + coverage_build_status = None + if project.name in coverage_build_statuses: + coverage_build_status = coverage_build_statuses[project.name] + + futures.append( + executor.submit(update_build_badges, project.name, + project_build_statuses[project.name], + coverage_build_status)) concurrent.futures.wait(futures) + + +def main(): + """Entry point for cloudbuild""" + with ndb.Client().context(): + configs = ((build_project.FUZZING_BUILD_TYPE, FUZZING_STATUS_FILENAME), + (build_and_run_coverage.COVERAGE_BUILD_TYPE, + COVERAGE_STATUS_FILENAME)) + + for tag, filename in configs: + update_build_status(tag, filename) + + update_badges() + + +if __name__ == '__main__': + main() diff --git a/infra/build/functions/update_build_status_test.py b/infra/build/build_status/update_build_status_test.py similarity index 100% rename from infra/build/functions/update_build_status_test.py rename to infra/build/build_status/update_build_status_test.py diff --git a/infra/build/functions/build_and_run_coverage.py b/infra/build/functions/build_and_run_coverage.py index 1195776d96e8..578d8d6ae90b 100755 --- a/infra/build/functions/build_and_run_coverage.py +++ b/infra/build/functions/build_and_run_coverage.py @@ -36,6 +36,7 @@ # Where code coverage reports need to be uploaded to. COVERAGE_BUCKET_NAME = 'oss-fuzz-coverage' +INTROSPECTOR_BUCKET_NAME = 'oss-fuzz-introspector' # This is needed for ClusterFuzz to pick up the most recent reports data. @@ -67,6 +68,28 @@ def get_upload_url(self, upload_type): f'/{upload_type}/{self.date}') +class IntrospectorBucket: # pylint: disable=too-few-public-methods + """Class representing the fuzz introspector GCS bucket.""" + + def __init__(self, project, date, platform, testing): + self.introspector_bucket_name = 'oss-fuzz-introspector' + if testing: + self.introspector_bucket_name += '-testing' + + self.date = date + self.project = project + self.html_report_url = ( + f'{build_lib.GCS_URL_BASENAME}{self.introspector_bucket_name}/{project}' + f'/reports/{date}/{platform}/index.html') + self.latest_report_info_url = (f'/{INTROSPECTOR_BUCKET_NAME}' + f'/latest_report_info/{project}.json') + + def get_upload_url(self, upload_type): + """Returns an upload url for |upload_type|.""" + return (f'gs://{self.introspector_bucket_name}/{self.project}' + f'/{upload_type}/{self.date}') + + def get_build_steps( # pylint: disable=too-many-locals, too-many-arguments project_name, project_yaml_contents, dockerfile_lines, image_project, base_images_project, config): @@ -121,11 +144,11 @@ def get_build_steps( # pylint: disable=too-many-locals, too-many-arguments coverage_env.append('FULL_SUMMARY_PER_TARGET=1') build_steps.append({ - 'name': - build_project.get_runner_image_name(base_images_project, - config.test_image_suffix), - 'env': - coverage_env, + 'name': # build_project.get_runner_image_name(base_images_project, + # config.test_image_suffix), + # 'gcr.io/oss-fuzz-base/base-runner-customized', #TODO: hacky! to be fixed + 'gcr.io/oss-fuzz-base/base-runner:introspector', + 'env': coverage_env, 'args': [ 'bash', '-c', ('for f in /corpus/*.zip; do unzip -q $f -d ${f%%.*} || (' @@ -223,6 +246,115 @@ def get_build_steps( # pylint: disable=too-many-locals, too-many-arguments build_lib.http_upload_step(latest_report_info_body, latest_report_info_url, LATEST_REPORT_INFO_CONTENT_TYPE)) + #removes index.html from the end of url + coverage_url = bucket.html_report_url[:-11] + build_steps.extend( + get_fuzz_introspector_steps(project, project_name, base_images_project, + config, coverage_url)) + return build_steps + + +def get_fuzz_introspector_steps(project, project_name, base_images_project, + config, coverage_url): + build_steps = [] + FI_dir = '/workspace/fuzz-introspector/' + oss_integration_dir = 'oss_fuzz_integration/' + + report_date = build_project.get_datetime_now().strftime('%Y%m%d') + bucket = IntrospectorBucket(project.name, report_date, PLATFORM, + config.testing) + + build = build_project.Build('libfuzzer', 'instrumentor', 'x86_64') + env = build_project.get_env(project.fuzzing_language, build) + + clone_step = { + 'args': [ + 'clone', 'https://github.com/ossf/fuzz-introspector.git', '--depth', + '1' + ], + 'name': 'gcr.io/cloud-builders/git', + } + build_steps.append(clone_step) + + build_steps.append({ + 'name': + build_project.get_runner_image_name(base_images_project, + config.test_image_suffix), + 'args': [ + 'bash', '-c', + (f'cd {FI_dir} && cd {oss_integration_dir}' + ' && sed -i \'s/\.\/infra\/base\-images\/all.sh/#\.\/infra\/base\-images\/all.sh/\'' + ' build_patched_oss_fuzz.sh' + ' && cat build_patched_oss_fuzz.sh' + ' && ./build_patched_oss_fuzz.sh') + ] + }) + + build_steps.append({ + 'name': + build_project.get_runner_image_name(base_images_project, + config.test_image_suffix), + 'args': [ + 'bash', '-c', + ('sed -i s/base-builder/base-builder:introspector/g ' + f'{FI_dir}{oss_integration_dir}oss-fuzz/projects/{project_name}/Dockerfile' + f' && cat {FI_dir}{oss_integration_dir}oss-fuzz/projects/{project_name}/Dockerfile' + ) + ] + }) + + build_steps.append({ + 'name': + 'gcr.io/cloud-builders/docker', + 'args': [ + 'build', + '-t', + f'gcr.io/oss-fuzz/{project_name}', + '--file', + f'{FI_dir}{oss_integration_dir}oss-fuzz/projects/{project_name}/Dockerfile', + f'{FI_dir}{oss_integration_dir}oss-fuzz/projects/{project_name}', + ], + }) + + build_steps.append( + build_project.get_compile_step(project, build, env, config.parallel)) + + #adjust coverage url + cov_url_escaped = coverage_url.replace("/", "\/").replace(":", "\:") + set_cov_url = ( + f'sed -i \'s/http\:\/\/localhost\:8008\/covreport\/linux/{cov_url_escaped}/\'' + ' /src/post-processing/main.py && cat /src/post-processing/main.py && ') + last_build_step = build_steps[-1] + last_args = last_build_step['args'] + last_bash_cmd = last_args[2] + last_bash_cmd = set_cov_url + last_bash_cmd + build_steps[-1]['args'][2] = last_bash_cmd + + build_steps.append({ + 'name': + build_project.get_runner_image_name(base_images_project, + config.test_image_suffix), + 'args': ['bash', '-c', ('du -a /workspace')] + }) + + # Upload the report. + upload_report_url = bucket.get_upload_url('inspector-report') + + # Delete the existing report as gsutil cannot overwrite it in a useful way due + # to the lack of `-T` option (it creates a subdir in the destination dir). + build_steps.append(build_lib.gsutil_rm_rf_step(upload_report_url)) + build_steps.append({ + 'name': + 'gcr.io/cloud-builders/gsutil', + 'args': [ + '-m', + 'cp', + '-r', + os.path.join(build.out, 'inspector-tmp'), + upload_report_url, + ], + }) + return build_steps diff --git a/infra/build/functions/build_project.py b/infra/build/functions/build_project.py index bdc7985e172f..12229de78cbd 100755 --- a/infra/build/functions/build_project.py +++ b/infra/build/functions/build_project.py @@ -195,12 +195,19 @@ def get_env(fuzzing_language, build): 'HOME': '/root', 'OUT': build.out, } + return list(sorted([f'{key}={value}' for key, value in env_dict.items()])) def get_compile_step(project, build, env, parallel): """Returns the GCB step for compiling |projects| fuzzers using |env|. The type of build is specified by |build|.""" + set_git_repo_env = '' #do nothing + if build.sanitizer == 'instrumentor': + set_git_repo_env = ( + ' && export GITHUB_REPO=$(grep -P -o "\S+github.com\S+" /workspace/oss-fuzz/projects/' + f'{project.name}/Dockerfile | xargs -L 1 | sed -e "s/.git$//") && set |grep GITHUB_REPO ' + ) failure_msg = ( '*' * 80 + '\nFailed to build.\nTo reproduce, run:\n' f'python infra/helper.py build_image {project.name}\n' @@ -218,7 +225,7 @@ def get_compile_step(project, build, env, parallel): # Dockerfile). Container Builder overrides our workdir so we need # to add this step to set it back. (f'rm -r /out && cd /src && cd {project.workdir} && ' - f'mkdir -p {build.out} && compile || ' + f'mkdir -p {build.out} {set_git_repo_env} && compile || ' f'(echo "{failure_msg}" && false)'), ], 'id': get_id('compile', build),