<commit_before>#!/usr/bin/python2

"""Starts project build on Google Cloud Builder.

Usage: build.py <project_dir>
"""

import base64
import collections
import datetime
import os
import re
import subprocess
import sys
import time
import urllib
import yaml

from oauth2client.client import GoogleCredentials
from oauth2client.service_account import ServiceAccountCredentials
from googleapiclient.discovery import build

BUILD_TIMEOUT = 10 * 60 * 60

CONFIGURATIONS = {
  'sanitizer-address' : [ 'SANITIZER=address' ],
  'sanitizer-memory' : [ 'SANITIZER=memory' ],
  'sanitizer-undefined' : [ 'SANITIZER=undefined' ],
  'sanitizer-coverage' : [ 'SANITIZER=coverage' ],
  'engine-libfuzzer' : [ 'FUZZING_ENGINE=libfuzzer' ],
  'engine-afl' : [ 'FUZZING_ENGINE=afl' ],
  'engine-honggfuzz' : [ 'FUZZING_ENGINE=honggfuzz' ],
  'engine-none' : [ 'FUZZING_ENGINE=none' ],
}

EngineInfo = collections.namedtuple(
    'EngineInfo', ['upload_bucket', 'supported_sanitizers'])

ENGINE_INFO = {
    'libfuzzer': EngineInfo(
        upload_bucket='clusterfuzz-builds',
        supported_sanitizers=['address', 'memory', 'undefined', 'coverage']),
    'afl': EngineInfo(
        upload_bucket='clusterfuzz-builds-afl',
        supported_sanitizers=['address']),
    'honggfuzz': EngineInfo(
        upload_bucket='clusterfuzz-builds-honggfuzz',
        supported_sanitizers=['address', 'memory', 'undefined']),
    'none': EngineInfo(
        upload_bucket='clusterfuzz-builds-no-engine',
        supported_sanitizers=['address']),
}

DEFAULT_ENGINES = ['libfuzzer', 'afl', 'honggfuzz']
DEFAULT_SANITIZERS = ['address', 'undefined']


def usage():
  sys.stderr.write(
    "Usage: " + sys.argv[0] + " <project_dir>\n")
  exit(1)


def load_project_yaml(project_dir):
  project_name = os.path.basename(project_dir)
  project_yaml_path = os.path.join(project_dir, 'project.yaml')
  with open(project_yaml_path) as f:
    project_yaml = yaml.safe_load(f)
    project_yaml.setdefault('name', project_name)
    project_yaml.setdefault('image',
        'gcr.io/oss-fuzz/' + project_name)
    project_yaml.setdefault('sanitizers', DEFAULT_SANITIZERS)
    project_yaml.setdefault('fuzzing_engines', DEFAULT_ENGINES)
    return project_yaml


def get_signed_url(path):
  timestamp = int(time.time() + BUILD_TIMEOUT)
  blob = 'PUT\n\n\n{0}\n{1}'.format(
      timestamp, path)

  creds = ServiceAccountCredentials.from_json_keyfile_name(
      os.environ['GOOGLE_APPLICATION_CREDENTIALS'])
  client_id = creds.service_account_email
  signature = base64.b64encode(creds.sign_blob(blob)[1])
  values = {
      'GoogleAccessId': client_id,
      'Expires': timestamp,
      'Signature': signature,
  }

  return ('https://storage.googleapis.com{0}?'.format(path) +
          urllib.urlencode(values))


def is_supported_configuration(fuzzing_engine, sanitizer):
  return sanitizer in ENGINE_INFO[fuzzing_engine].supported_sanitizers


def get_sanitizers(project_yaml):
  sanitizers = project_yaml['sanitizers']
  assert isinstance(sanitizers, list)

  processed_sanitizers = []
  for sanitizer in sanitizers:
    if isinstance(sanitizer, basestring):
      processed_sanitizers.append(sanitizer)
    elif isinstance(sanitizer, dict):
      for key in sanitizer.iterkeys():
        processed_sanitizers.append(key)

  # Always make a coverage build.
  if 'coverage' not in processed_sanitizers:
    processed_sanitizers.append('coverage')

  return processed_sanitizers


def workdir_from_dockerfile(dockerfile):
  """Parse WORKDIR from the Dockerfile."""
  WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)')

  with open(dockerfile) as f:
    lines = f.readlines()

  for line in lines:
    match = re.match(WORKDIR_REGEX, line)
    if match:
      # We need to escape '$' since they're used for subsitutions in Container
      # Builer builds.
      return match.group(1).replace('$', '$$')

  return None


def get_build_steps(project_yaml, dockerfile_path):
  name = project_yaml['name']
  image = project_yaml['image']

  ts = datetime.datetime.now().strftime('%Y%m%d%H%M')

  build_steps = [
      {
          'args': [
              'clone', 'https://github.com/google/oss-fuzz.git',
          ],
          'name': 'gcr.io/cloud-builders/git',
      },
      {
          'name': 'gcr.io/cloud-builders/docker',
          'args': [
              'build',
              '-t',
              image,
              '.',
          ],
          'dir': 'oss-fuzz/projects/' + name,
      },
      {
          'name': image,
          'args': [
            'bash',
            '-c',
            'srcmap > /workspace/srcmap.json && cat /workspace/srcmap.json'
          ],
          'env': [ 'OSSFUZZ_REVISION=$REVISION_ID' ],
      },
      {
          'name': 'gcr.io/oss-fuzz-base/msan-builder',
          'args': [
            'bash',
            '-c',
            'cp -r /msan /workspace',
          ],
      },
  ]

  for fuzzing_engine in project_yaml['fuzzing_engines']:
    for sanitizer in get_sanitizers(project_yaml):
      if not is_supported_configuration(fuzzing_engine, sanitizer):
        continue

      env = CONFIGURATIONS['engine-' + fuzzing_engine][:]
      env.extend(CONFIGURATIONS['sanitizer-' + sanitizer])
      out = '/workspace/out/' + sanitizer
      stamped_name = name + '-' + sanitizer + '-' + ts
      zip_file = stamped_name + '.zip'
      stamped_srcmap_file = stamped_name + '.srcmap.json'
      bucket = ENGINE_INFO[fuzzing_engine].upload_bucket
      upload_url = get_signed_url('/{0}/{1}/{2}'.format(
          bucket, name, zip_file))
      srcmap_url = get_signed_url('/{0}/{1}/{2}'.format(
          bucket, name, stamped_srcmap_file))

      env.append('OUT=' + out)
      env.append('MSAN_LIBS_PATH=/workspace/msan')

      workdir = workdir_from_dockerfile(dockerfile_path)
      if not workdir:
        workdir = '/src'

      build_steps.extend([
          # compile
          {'name': image,
           'env': env,
           'args': [
             'bash',
             '-c',
             # Remove /out to break loudly when a build script incorrectly uses
             # /out instead of $OUT.
             # `cd /src && cd {workdir}` (where {workdir} is parsed from the
             # Dockerfile). Container Builder overrides our workdir so we need to add
             # this step to set it back.
             # We also remove /work and /src to save disk space after a step.
             # Container Builder doesn't pass --rm to docker run yet.
             'rm -r /out && cd /src && cd {1} && mkdir -p {0} && compile && rm -rf /work && rm -rf /src'.format(out, workdir),
           ],
          },
          # test binaries
          {'name': 'gcr.io/oss-fuzz-base/base-runner',
            'env': env,
            'args': [
              'bash',
              '-c',
              # Verify that fuzzers have been built properly and are not broken.
              # TODO(mmoroz): raise a notification if not passing the tests.
              'test_all'
            ],
          },
      ])

      if sanitizer == 'memory':
        # Patch dynamic libraries to use instrumented ones.
        build_steps.append({
          'name': 'gcr.io/oss-fuzz-base/msan-builder',
          'args': [
            'bash',
            '-c',
            # TODO(ochang): Replace with just patch_build.py once permission in
            # image is fixed.<commit_msg>Speculative fix for firefox build fix. (#1661)<commit_after>#!/usr/bin/python2

"""Starts project build on Google Cloud Builder.

Usage: build.py <project_dir>
"""

import base64
import collections
import datetime
import os
import re
import subprocess
import sys
import time
import urllib
import yaml

from oauth2client.client import GoogleCredentials
from oauth2client.service_account import ServiceAccountCredentials
from googleapiclient.discovery import build

BUILD_TIMEOUT = 10 * 60 * 60

CONFIGURATIONS = {
  'sanitizer-address' : [ 'SANITIZER=address' ],
  'sanitizer-memory' : [ 'SANITIZER=memory' ],
  'sanitizer-undefined' : [ 'SANITIZER=undefined' ],
  'sanitizer-coverage' : [ 'SANITIZER=coverage' ],
  'engine-libfuzzer' : [ 'FUZZING_ENGINE=libfuzzer' ],
  'engine-afl' : [ 'FUZZING_ENGINE=afl' ],
  'engine-honggfuzz' : [ 'FUZZING_ENGINE=honggfuzz' ],
  'engine-none' : [ 'FUZZING_ENGINE=none' ],
}

EngineInfo = collections.namedtuple(
    'EngineInfo', ['upload_bucket', 'supported_sanitizers'])

ENGINE_INFO = {
    'libfuzzer': EngineInfo(
        upload_bucket='clusterfuzz-builds',
        supported_sanitizers=['address', 'memory', 'undefined', 'coverage']),
    'afl': EngineInfo(
        upload_bucket='clusterfuzz-builds-afl',
        supported_sanitizers=['address']),
    'honggfuzz': EngineInfo(
        upload_bucket='clusterfuzz-builds-honggfuzz',
        supported_sanitizers=['address', 'memory', 'undefined']),
    'none': EngineInfo(
        upload_bucket='clusterfuzz-builds-no-engine',
        supported_sanitizers=['address']),
}

DEFAULT_ENGINES = ['libfuzzer', 'afl', 'honggfuzz']
DEFAULT_SANITIZERS = ['address', 'undefined']


def usage():
  sys.stderr.write(
    "Usage: " + sys.argv[0] + " <project_dir>\n")
  exit(1)


def load_project_yaml(project_dir):
  project_name = os.path.basename(project_dir)
  project_yaml_path = os.path.join(project_dir, 'project.yaml')
  with open(project_yaml_path) as f:
    project_yaml = yaml.safe_load(f)
    project_yaml.setdefault('name', project_name)
    project_yaml.setdefault('image',
        'gcr.io/oss-fuzz/' + project_name)
    project_yaml.setdefault('sanitizers', DEFAULT_SANITIZERS)
    project_yaml.setdefault('fuzzing_engines', DEFAULT_ENGINES)
    project_yaml.setdefault('run_tests', True)
    return project_yaml


def get_signed_url(path):
  timestamp = int(time.time() + BUILD_TIMEOUT)
  blob = 'PUT\n\n\n{0}\n{1}'.format(
      timestamp, path)

  creds = ServiceAccountCredentials.from_json_keyfile_name(
      os.environ['GOOGLE_APPLICATION_CREDENTIALS'])
  client_id = creds.service_account_email
  signature = base64.b64encode(creds.sign_blob(blob)[1])
  values = {
      'GoogleAccessId': client_id,
      'Expires': timestamp,
      'Signature': signature,
  }

  return ('https://storage.googleapis.com{0}?'.format(path) +
          urllib.urlencode(values))


def is_supported_configuration(fuzzing_engine, sanitizer):
  return sanitizer in ENGINE_INFO[fuzzing_engine].supported_sanitizers


def get_sanitizers(project_yaml):
  sanitizers = project_yaml['sanitizers']
  assert isinstance(sanitizers, list)

  processed_sanitizers = []
  for sanitizer in sanitizers:
    if isinstance(sanitizer, basestring):
      processed_sanitizers.append(sanitizer)
    elif isinstance(sanitizer, dict):
      for key in sanitizer.iterkeys():
        processed_sanitizers.append(key)

  # Always make a coverage build.
  if 'coverage' not in processed_sanitizers:
    processed_sanitizers.append('coverage')

  return processed_sanitizers


def workdir_from_dockerfile(dockerfile):
  """Parse WORKDIR from the Dockerfile."""
  WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)')

  with open(dockerfile) as f:
    lines = f.readlines()

  for line in lines:
    match = re.match(WORKDIR_REGEX, line)
    if match:
      # We need to escape '$' since they're used for subsitutions in Container
      # Builer builds.
      return match.group(1).replace('$', '$$')

  return None


def get_build_steps(project_yaml, dockerfile_path):
  name = project_yaml['name']
  image = project_yaml['image']
  run_tests = project_yaml['run_tests']

  ts = datetime.datetime.now().strftime('%Y%m%d%H%M')

  build_steps = [
      {
          'args': [
              'clone', 'https://github.com/google/oss-fuzz.git',
          ],
          'name': 'gcr.io/cloud-builders/git',
      },
      {
          'name': 'gcr.io/cloud-builders/docker',
          'args': [
              'build',
              '-t',
              image,
              '.',
          ],
          'dir': 'oss-fuzz/projects/' + name,
      },
      {
          'name': image,
          'args': [
            'bash',
            '-c',
            'srcmap > /workspace/srcmap.json && cat /workspace/srcmap.json'
          ],
          'env': [ 'OSSFUZZ_REVISION=$REVISION_ID' ],
      },
      {
          'name': 'gcr.io/oss-fuzz-base/msan-builder',
          'args': [
            'bash',
            '-c',
            'cp -r /msan /workspace',
          ],
      },
  ]

  for fuzzing_engine in project_yaml['fuzzing_engines']:
    for sanitizer in get_sanitizers(project_yaml):
      if not is_supported_configuration(fuzzing_engine, sanitizer):
        continue

      env = CONFIGURATIONS['engine-' + fuzzing_engine][:]
      env.extend(CONFIGURATIONS['sanitizer-' + sanitizer])
      out = '/workspace/out/' + sanitizer
      stamped_name = name + '-' + sanitizer + '-' + ts
      zip_file = stamped_name + '.zip'
      stamped_srcmap_file = stamped_name + '.srcmap.json'
      bucket = ENGINE_INFO[fuzzing_engine].upload_bucket
      upload_url = get_signed_url('/{0}/{1}/{2}'.format(
          bucket, name, zip_file))
      srcmap_url = get_signed_url('/{0}/{1}/{2}'.format(
          bucket, name, stamped_srcmap_file))

      env.append('OUT=' + out)
      env.append('MSAN_LIBS_PATH=/workspace/msan')

      workdir = workdir_from_dockerfile(dockerfile_path)
      if not workdir:
        workdir = '/src'

      build_steps.append(
          # compile
          {'name': image,
           'env': env,
           'args': [
             'bash',
             '-c',
             # Remove /out to break loudly when a build script incorrectly uses
             # /out instead of $OUT.
             # `cd /src && cd {workdir}` (where {workdir} is parsed from the
             # Dockerfile). Container Builder overrides our workdir so we need to add
             # this step to set it back.
             # We also remove /work and /src to save disk space after a step.
             # Container Builder doesn't pass --rm to docker run yet.
             'rm -r /out && cd /src && cd {1} && mkdir -p {0} && compile && rm -rf /work && rm -rf /src'.format(out, workdir),
           ],
          }
      )

      if run_tests:
        build_steps.append(
            # test binaries
            {'name': 'gcr.io/oss-fuzz-base/base-runner',
              'env': env,
              'args': [
                'bash',
                '-c',
                # Verify that fuzzers are built properly and not broken.
                # TODO(mmoroz): raise a notification on failing tests.
                'test_all'
              ],
            }
        )

      if sanitizer == 'memory':
        # Patch dynamic libraries to use instrumented ones.
        build_steps.append({
          'name': 'gcr.io/oss-fuzz-base/msan-builder',
          'args': [
            'bash',
            '-c',
            # TODO(ochang): Replace with just patch_build.py once permission in
            # image is fixed.
{'fraction_non_alphanumeric': 0.1318349547131835, 'fraction_numerical': 0.0042938611204293865, 'mean_word_length': 2.595272551857212, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7416926', 'n_tokens_mistral': 4986, 'n_tokens_neox': 4801, 'n_words': 978}