<commit_before>#!/usr/bin/env python
#
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Gets a Chromium archived build, and unpacks it
   into a target directory.

  Use -r option to specify the revison number
  Use -t option to specify the directory to unzip the build into.

Usage:
  $ get_chromium_build.py -r <revision> -t <target>
"""

import logging
import optparse
import os
import platform
import shutil
import subprocess
import sys
import time
import urllib
import urllib2
import zipfile

# Example chromium build location:
# gs://chromium-browser-snapshots/Linux_x64/228977/chrome-linux.zip
CHROMIUM_URL_FMT = ('http://commondatastorage.googleapis.com/'
                    'chromium-browser-snapshots/%s/%s/%s')

class BuildUpdater(object):
  _PLATFORM_PATHS_MAP = {
      'Linux': { 'zipfiles': ['chrome-linux.zip'],
                 'folder': 'chrome_linux',
                 'archive_path': 'Linux_x64'},
      'Darwin': {'zipfiles': ['chrome-mac.zip'],
                 'folder': 'chrome_mac',
                 'archive_path': 'Mac'},
      'Windows': {'zipfiles': ['chrome-win32.zip',
                               'chrome-win32-syms.zip'],
                 'folder': 'chrome_win',
                 'archive_path': 'Win'}}

  def __init__(self, options):
    platform_data = BuildUpdater._PLATFORM_PATHS_MAP[platform.system()]
    self._zipfiles = platform_data['zipfiles']
    self._folder = platform_data['folder']
    self._archive_path = platform_data['archive_path']
    self._revision = int(options.revision)
    self._target_dir = options.target_dir
    self._download_dir = os.path.join(self._target_dir, 'downloads')

  def _GetBuildUrl(self, revision, filename):
    return CHROMIUM_URL_FMT % (self._archive_path, revision, filename)

  def _FindBuildRevision(self, revision, filename):
    MAX_REVISIONS_PER_BUILD = 100
    for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD):
      if self._DoesBuildExist(revision_guess, filename):
        return revision_guess
      else:
        time.sleep(.1)
    return None

  def _DoesBuildExist(self, revision_guess, filename):
    url = self._GetBuildUrl(revision_guess, filename)

    r = urllib2.Request(url)
    r.get_method = lambda: 'HEAD'
    try:
      urllib2.urlopen(r)
      return True
    except urllib2.HTTPError, err:
      if err.code == 404:
        return False

  def _DownloadBuild(self):
    if not os.path.exists(self._download_dir):<commit_msg>Multivm branch: Fetch reference build from archive using git hash, not revision.<commit_after>#!/usr/bin/env python
#
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Gets a Chromium archived build, and unpacks it
   into a target directory.

  Use -r option to specify the revison number
  Use -t option to specify the directory to unzip the build into.

Usage:
  $ get_chromium_build.py -r <revision> -t <target>
"""

import json
import logging
import optparse
import os
import platform
import shutil
import subprocess
import sys
import time
import urllib
import urllib2
import zipfile

# Example chromium build location:
# gs://chromium-browser-snapshots/Linux_x64/228977/chrome-linux.zip
CHROMIUM_URL_FMT = ('http://commondatastorage.googleapis.com/'
                    'chromium-browser-snapshots/%s/%s/%s')

class BuildUpdater(object):
  _PLATFORM_PATHS_MAP = {
      'Linux': { 'zipfiles': ['chrome-linux.zip'],
                 'folder': 'chrome_linux',
                 'archive_path': 'Linux_x64'},
      'Darwin': {'zipfiles': ['chrome-mac.zip'],
                 'folder': 'chrome_mac',
                 'archive_path': 'Mac'},
      'Windows': {'zipfiles': ['chrome-win32.zip',
                               'chrome-win32-syms.zip'],
                 'folder': 'chrome_win',
                 'archive_path': 'Win'}}

  def __init__(self, options):
    platform_data = BuildUpdater._PLATFORM_PATHS_MAP[platform.system()]
    self._zipfiles = platform_data['zipfiles']
    self._folder = platform_data['folder']
    self._archive_path = platform_data['archive_path']
    self._revision = options.revision
    self._target_dir = options.target_dir
    self._download_dir = os.path.join(self._target_dir, 'downloads')

  def _GetBuildUrl(self, revision, filename):
    return CHROMIUM_URL_FMT % (self._archive_path, revision, filename)

  def _FindBuildRevision(self, revision, filename):
    git_hash = json.loads(revision)[platform.system()]
    if self._DoesBuildExist(git_hash, filename):
      return git_hash
    return None

  def _DoesBuildExist(self, revision_guess, filename):
    url = self._GetBuildUrl(revision_guess, filename)

    r = urllib2.Request(url)
    r.get_method = lambda: 'HEAD'
    try:
      urllib2.urlopen(r)
      return True
    except urllib2.HTTPError, err:
      if err.code == 404:
        return False

  def _DownloadBuild(self):
    if not os.path.exists(self._download_dir):
{'fraction_non_alphanumeric': 0.10978939157566303, 'fraction_numerical': 0.010530421216848674, 'mean_word_length': 3.19721767594108, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6253523', 'n_tokens_mistral': 1656, 'n_tokens_neox': 1593, 'n_words': 404}