Adding Fuzz Introspector to cloud build

infra/build/build_status/Dockerfile

@@ -0,0 +1,24 @@
+# Copyright 2021 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+FROM gcr.io/oss-fuzz-base/base-runner
+
+RUN mkdir -p /opt/oss-fuzz/infra/build_status
+COPY ${OSS_FUZZ_ROOT}/infra/build/functions/* /opt/oss-fuzz/infra/build_status/
+COPY ${OSS_FUZZ_ROOT}/infra/build/build_status/* /opt/oss-fuzz/infra/build_status/
+RUN pip install -r /opt/oss-fuzz/infra/build_status/requirements.txt
+
+ENTRYPOINT [ "python3", "/opt/oss-fuzz/infra/build_status/update_build_status.py" ]

infra/build/build_status/cloudbuild.yaml

@@ -0,0 +1,15 @@
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  args:
+  - build
+  - '-t'
+  - gcr.io/oss-fuzz-base/build-status
+  - '-f'
+  - infra/build/build_status/Dockerfile
+  - .
+- name: 'gcr.io/oss-fuzz-base/build-status'
+  args: []
+  timeout: 3600s
+options:
+  logging: CLOUD_LOGGING_ONLY
+timeout: 3600s

infra/build/functions/update_build_status.py → infra/build/build_status/update_build_status.py

@@ -14,7 +14,6 @@
 #
 ################################################################################
 """Cloud function to request builds."""
-import base64
 import concurrent.futures
 import json
 import sys
@@ -239,33 +238,6 @@ def upload_log(build_id):
   return True
 
 
-# pylint: disable=no-member
-def update_status(event, context):
-  """Entry point for cloud function to update build statuses and badges."""
-  del context
-
-  if 'data' in event:
-    status_type = base64.b64decode(event['data']).decode()
-  else:
-    raise RuntimeError('No data')
-
-  if status_type == 'badges':
-    update_badges()
-    return
-
-  if status_type == 'fuzzing':
-    tag = build_project.FUZZING_BUILD_TYPE
-    status_filename = FUZZING_STATUS_FILENAME
-  elif status_type == 'coverage':
-    tag = build_and_run_coverage.COVERAGE_BUILD_TYPE
-    status_filename = COVERAGE_STATUS_FILENAME
-  else:
-    raise RuntimeError('Invalid build status type ' + status_type)
-
-  with ndb.Client().context():
-    update_build_status(tag, status_filename)
-
-
 def load_status_from_gcs(filename):
   """Load statuses from bucket."""
   status_bucket = get_storage_client().get_bucket(STATUS_BUCKET)
@@ -286,18 +258,34 @@ def update_badges():
 
   with concurrent.futures.ThreadPoolExecutor(max_workers=32) as executor:
     futures = []
-    with ndb.Client().context():
-      for project in Project.query():
-        if project.name not in project_build_statuses:
-          continue
-        # Certain projects (e.g. JVM and Python) do not have any coverage
-        # builds, but should still receive a badge.
-        coverage_build_status = None
-        if project.name in coverage_build_statuses:
-          coverage_build_status = coverage_build_statuses[project.name]
-
-        futures.append(
-            executor.submit(update_build_badges, project.name,
-                            project_build_statuses[project.name],
-                            coverage_build_status))
+    for project in Project.query():
+      if project.name not in project_build_statuses:
+        continue
+      # Certain projects (e.g. JVM and Python) do not have any coverage
+      # builds, but should still receive a badge.
+      coverage_build_status = None
+      if project.name in coverage_build_statuses:
+        coverage_build_status = coverage_build_statuses[project.name]
+
+      futures.append(
+          executor.submit(update_build_badges, project.name,
+                          project_build_statuses[project.name],
+                          coverage_build_status))
     concurrent.futures.wait(futures)
+
+
+def main():
+  """Entry point for cloudbuild"""
+  with ndb.Client().context():
+    configs = ((build_project.FUZZING_BUILD_TYPE, FUZZING_STATUS_FILENAME),
+               (build_and_run_coverage.COVERAGE_BUILD_TYPE,
+                COVERAGE_STATUS_FILENAME))
+
+    for tag, filename in configs:
+      update_build_status(tag, filename)
+
+    update_badges()
+
+
+if __name__ == '__main__':
+  main()

infra/build/functions/build_and_run_coverage.py

@@ -36,6 +36,7 @@
 
 # Where code coverage reports need to be uploaded to.
 COVERAGE_BUCKET_NAME = 'oss-fuzz-coverage'
+INTROSPECTOR_BUCKET_NAME = 'oss-fuzz-introspector'
 
 # This is needed for ClusterFuzz to pick up the most recent reports data.
 
@@ -67,6 +68,28 @@ def get_upload_url(self, upload_type):
             f'/{upload_type}/{self.date}')
 
 
+class IntrospectorBucket:  # pylint: disable=too-few-public-methods
+  """Class representing the fuzz introspector GCS bucket."""
+
+  def __init__(self, project, date, platform, testing):
+    self.introspector_bucket_name = 'oss-fuzz-introspector'
+    if testing:
+      self.introspector_bucket_name += '-testing'
+
+    self.date = date
+    self.project = project
+    self.html_report_url = (
+        f'{build_lib.GCS_URL_BASENAME}{self.introspector_bucket_name}/{project}'
+        f'/reports/{date}/{platform}/index.html')
+    self.latest_report_info_url = (f'/{INTROSPECTOR_BUCKET_NAME}'
+                                   f'/latest_report_info/{project}.json')
+
+  def get_upload_url(self, upload_type):
+    """Returns an upload url for |upload_type|."""
+    return (f'gs://{self.introspector_bucket_name}/{self.project}'
+            f'/{upload_type}/{self.date}')
+
+
 def get_build_steps(  # pylint: disable=too-many-locals, too-many-arguments
     project_name, project_yaml_contents, dockerfile_lines, image_project,
     base_images_project, config):
@@ -121,11 +144,11 @@ def get_build_steps(  # pylint: disable=too-many-locals, too-many-arguments
     coverage_env.append('FULL_SUMMARY_PER_TARGET=1')
 
   build_steps.append({
-      'name':
-          build_project.get_runner_image_name(base_images_project,
-                                              config.test_image_suffix),
-      'env':
-          coverage_env,
+      'name':  #   build_project.get_runner_image_name(base_images_project,
+          #                                       config.test_image_suffix),
+          # 'gcr.io/oss-fuzz-base/base-runner-customized', #TODO: hacky! to be fixed
+      'gcr.io/oss-fuzz-base/base-runner:introspector',
+      'env': coverage_env,
       'args': [
           'bash', '-c',
           ('for f in /corpus/*.zip; do unzip -q $f -d ${f%%.*} || ('
@@ -223,6 +246,115 @@ def get_build_steps(  # pylint: disable=too-many-locals, too-many-arguments
       build_lib.http_upload_step(latest_report_info_body,
                                  latest_report_info_url,
                                  LATEST_REPORT_INFO_CONTENT_TYPE))
+  #removes index.html from the end of url
+  coverage_url = bucket.html_report_url[:-11]
+  build_steps.extend(
+      get_fuzz_introspector_steps(project, project_name, base_images_project,
+                                  config, coverage_url))
+  return build_steps
+
+
+def get_fuzz_introspector_steps(project, project_name, base_images_project,
+                                config, coverage_url):
+  build_steps = []
+  FI_dir = '/workspace/fuzz-introspector/'
+  oss_integration_dir = 'oss_fuzz_integration/'
+
+  report_date = build_project.get_datetime_now().strftime('%Y%m%d')
+  bucket = IntrospectorBucket(project.name, report_date, PLATFORM,
+                              config.testing)
+
+  build = build_project.Build('libfuzzer', 'instrumentor', 'x86_64')
+  env = build_project.get_env(project.fuzzing_language, build)
+
+  clone_step = {
+      'args': [
+          'clone', 'https://github.com/ossf/fuzz-introspector.git', '--depth',
+          '1'
+      ],
+      'name': 'gcr.io/cloud-builders/git',
+  }
+  build_steps.append(clone_step)
+
+  build_steps.append({
+      'name':
+          build_project.get_runner_image_name(base_images_project,
+                                              config.test_image_suffix),
+      'args': [
+          'bash', '-c',
+          (f'cd {FI_dir} && cd {oss_integration_dir}'
+           ' && sed -i \'s/\.\/infra\/base\-images\/all.sh/#\.\/infra\/base\-images\/all.sh/\''
+           ' build_patched_oss_fuzz.sh'
+           ' && cat build_patched_oss_fuzz.sh'
+           ' && ./build_patched_oss_fuzz.sh')
+      ]
+  })
+
+  build_steps.append({
+      'name':
+          build_project.get_runner_image_name(base_images_project,
+                                              config.test_image_suffix),
+      'args': [
+          'bash', '-c',
+          ('sed -i s/base-builder/base-builder:introspector/g '
+           f'{FI_dir}{oss_integration_dir}oss-fuzz/projects/{project_name}/Dockerfile'
+           f' && cat {FI_dir}{oss_integration_dir}oss-fuzz/projects/{project_name}/Dockerfile'
+          )
+      ]
+  })
+
+  build_steps.append({
+      'name':
+          'gcr.io/cloud-builders/docker',
+      'args': [
+          'build',
+          '-t',
+          f'gcr.io/oss-fuzz/{project_name}',
+          '--file',
+          f'{FI_dir}{oss_integration_dir}oss-fuzz/projects/{project_name}/Dockerfile',
+          f'{FI_dir}{oss_integration_dir}oss-fuzz/projects/{project_name}',
+      ],
+  })
+
+  build_steps.append(
+      build_project.get_compile_step(project, build, env, config.parallel))
+
+  #adjust coverage url
+  cov_url_escaped = coverage_url.replace("/", "\/").replace(":", "\:")
+  set_cov_url = (
+      f'sed -i \'s/http\:\/\/localhost\:8008\/covreport\/linux/{cov_url_escaped}/\''
+      ' /src/post-processing/main.py && cat /src/post-processing/main.py && ')
+  last_build_step = build_steps[-1]
+  last_args = last_build_step['args']
+  last_bash_cmd = last_args[2]
+  last_bash_cmd = set_cov_url + last_bash_cmd
+  build_steps[-1]['args'][2] = last_bash_cmd
+
+  build_steps.append({
+      'name':
+          build_project.get_runner_image_name(base_images_project,
+                                              config.test_image_suffix),
+      'args': ['bash', '-c', ('du -a /workspace')]
+  })
+
+  # Upload the report.
+  upload_report_url = bucket.get_upload_url('inspector-report')
+
+  # Delete the existing report as gsutil cannot overwrite it in a useful way due
+  # to the lack of `-T` option (it creates a subdir in the destination dir).
+  build_steps.append(build_lib.gsutil_rm_rf_step(upload_report_url))
+  build_steps.append({
+      'name':
+          'gcr.io/cloud-builders/gsutil',
+      'args': [
+          '-m',
+          'cp',
+          '-r',
+          os.path.join(build.out, 'inspector-tmp'),
+          upload_report_url,
+      ],
+  })
+
   return build_steps
 
 

infra/build/functions/build_project.py

@@ -195,12 +195,19 @@ def get_env(fuzzing_language, build):
       'HOME': '/root',
       'OUT': build.out,
   }
+
   return list(sorted([f'{key}={value}' for key, value in env_dict.items()]))
 
 
 def get_compile_step(project, build, env, parallel):
   """Returns the GCB step for compiling |projects| fuzzers using |env|. The type
   of build is specified by |build|."""
+  set_git_repo_env = ''  #do nothing
+  if build.sanitizer == 'instrumentor':
+    set_git_repo_env = (
+        ' && export GITHUB_REPO=$(grep -P -o "\S+github.com\S+" /workspace/oss-fuzz/projects/'
+        f'{project.name}/Dockerfile | xargs -L 1 | sed -e "s/.git$//") && set |grep GITHUB_REPO '
+    )
   failure_msg = (
       '*' * 80 + '\nFailed to build.\nTo reproduce, run:\n'
       f'python infra/helper.py build_image {project.name}\n'
@@ -218,7 +225,7 @@ def get_compile_step(project, build, env, parallel):
           # Dockerfile). Container Builder overrides our workdir so we need
           # to add this step to set it back.
           (f'rm -r /out && cd /src && cd {project.workdir} && '
-           f'mkdir -p {build.out} && compile || '
+           f'mkdir -p {build.out} {set_git_repo_env} && compile || '
            f'(echo "{failure_msg}" && false)'),
       ],
       'id': get_id('compile', build),