<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Download files from Google Storage based on SHA1 sums."""


import hashlib
import optparse
import os
import Queue
import re
import stat
import sys
import threading
import time

import subprocess2


GSUTIL_DEFAULT_PATH = os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    'third_party', 'gsutil', 'gsutil')
# Maps sys.platform to what we actually want to call them.
PLATFORM_MAPPING = {
    'cygwin': 'win',
    'darwin': 'mac',
    'linux2': 'linux',
    'win32': 'win',
}


class FileNotFoundError(IOError):
  pass


class InvalidFileError(IOError):
  pass


class InvalidPlatformError(Exception):
  pass


def GetNormalizedPlatform():
  """Returns the result of sys.platform accounting for cygwin.
  Under cygwin, this will always return "win32" like the native Python."""
  if sys.platform == 'cygwin':
    return 'win32'
  return sys.platform


# Common utilities
class Gsutil(object):
  """Call gsutil with some predefined settings.  This is a convenience object,
  and is also immutable."""
  def __init__(self, path, boto_path, timeout=None, bypass_prodaccess=False):
    if not os.path.exists(path):
      raise FileNotFoundError('GSUtil not found in %s' % path)
    self.path = path
    self.timeout = timeout
    self.boto_path = boto_path
    self.bypass_prodaccess = bypass_prodaccess

  def get_sub_env(self):
    env = os.environ.copy()
    if self.boto_path == os.devnull:
      env['AWS_CREDENTIAL_FILE'] = ''
      env['BOTO_CONFIG'] = ''
    elif self.boto_path:
      env['AWS_CREDENTIAL_FILE'] = self.boto_path
      env['BOTO_CONFIG'] = self.boto_path
    else:
      custompath = env.get('AWS_CREDENTIAL_FILE', '~/.boto') + '.depot_tools'
      custompath = os.path.expanduser(custompath)
      if os.path.exists(custompath):
        env['AWS_CREDENTIAL_FILE'] = custompath

    return env

  def call(self, *args):
    cmd = [sys.executable, self.path]
    if self.bypass_prodaccess:
      cmd.append('--bypass_prodaccess')
    cmd.extend(args)
    return subprocess2.call(cmd, env=self.get_sub_env(), timeout=self.timeout)

  def check_call(self, *args):
    cmd = [sys.executable, self.path]
    if self.bypass_prodaccess:
      cmd.append('--bypass_prodaccess')
    cmd.extend(args)
    ((out, err), code) = subprocess2.communicate(
        cmd,
        stdout=subprocess2.PIPE,
        stderr=subprocess2.PIPE,
        env=self.get_sub_env(),
        timeout=self.timeout)

    # Parse output.
    status_code_match = re.search('status=([0-9]+)', err)
    if status_code_match:
      return (int(status_code_match.group(1)), out, err)
    if ('You are attempting to access protected data with '
          'no configured credentials.' in err):
      return (403, out, err)
    if 'No such object' in err:
      return (404, out, err)
    return (code, out, err)


def check_bucket_permissions(base_url, gsutil):
  code, _, ls_err = gsutil.check_call('ls', base_url)
  if code != 0:
    print >> sys.stderr, ls_err
  if code == 403:
    print >> sys.stderr, 'Got error 403 while authenticating to %s.' % base_url
    print >> sys.stderr, 'Try running "download_from_google_storage --config".'
  elif code == 404:
    print >> sys.stderr, '%s not found.' % base_url
  return (base_url, code)


def check_platform(target):
  """Checks if any parent directory of target matches (win|mac|linux)."""
  assert os.path.isabs(target)
  root, target_name = os.path.split(target)
  if not target_name:
    return None
  if target_name in ('linux', 'mac', 'win'):
    return target_name
  return check_platform(root)


def get_sha1(filename):
  sha1 = hashlib.sha1()
  with open(filename, 'rb') as f:
    while True:
      # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
      chunk = f.read(1024*1024)
      if not chunk:
        break
      sha1.update(chunk)
  return sha1.hexdigest()


# Download-specific code starts here

def enumerate_work_queue(input_filename, work_queue, directory,
                         recursive, ignore_errors, output, sha1_file,
                         auto_platform):
  if sha1_file:
    if not os.path.exists(input_filename):
      if not ignore_errors:
        raise FileNotFoundError('%s not found.' % input_filename)
      print >> sys.stderr, '%s not found.' % input_filename
    with open(input_filename, 'rb') as f:
      sha1_match = re.match('^([A-Za-z0-9]{40})$', f.read(1024).rstrip())
      if sha1_match:
        work_queue.put(
            (sha1_match.groups(1)[0], input_filename.replace('.sha1', '')))
        return 1
    if not ignore_errors:
      raise InvalidFileError('No sha1 sum found in %s.' % input_filename)
    print >> sys.stderr, 'No sha1 sum found in %s.' % input_filename
    return 0

  if not directory:
    work_queue.put((input_filename, output))
    return 1

  work_queue_size = 0
  for root, dirs, files in os.walk(input_filename):
    if not recursive:
      for item in dirs[:]:
        dirs.remove(item)
    else:
      for exclude in ['.svn', '.git']:
        if exclude in dirs:
          dirs.remove(exclude)
    for filename in files:
      full_path = os.path.join(root, filename)
      if full_path.endswith('.sha1'):
        if auto_platform:
          # Skip if the platform does not match.<commit_msg>Make --output and --sha1_filename work well together.<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Download files from Google Storage based on SHA1 sums."""


import hashlib
import optparse
import os
import Queue
import re
import stat
import sys
import threading
import time

import subprocess2


GSUTIL_DEFAULT_PATH = os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    'third_party', 'gsutil', 'gsutil')
# Maps sys.platform to what we actually want to call them.
PLATFORM_MAPPING = {
    'cygwin': 'win',
    'darwin': 'mac',
    'linux2': 'linux',
    'win32': 'win',
}


class FileNotFoundError(IOError):
  pass


class InvalidFileError(IOError):
  pass


class InvalidPlatformError(Exception):
  pass


def GetNormalizedPlatform():
  """Returns the result of sys.platform accounting for cygwin.
  Under cygwin, this will always return "win32" like the native Python."""
  if sys.platform == 'cygwin':
    return 'win32'
  return sys.platform


# Common utilities
class Gsutil(object):
  """Call gsutil with some predefined settings.  This is a convenience object,
  and is also immutable."""
  def __init__(self, path, boto_path, timeout=None, bypass_prodaccess=False):
    if not os.path.exists(path):
      raise FileNotFoundError('GSUtil not found in %s' % path)
    self.path = path
    self.timeout = timeout
    self.boto_path = boto_path
    self.bypass_prodaccess = bypass_prodaccess

  def get_sub_env(self):
    env = os.environ.copy()
    if self.boto_path == os.devnull:
      env['AWS_CREDENTIAL_FILE'] = ''
      env['BOTO_CONFIG'] = ''
    elif self.boto_path:
      env['AWS_CREDENTIAL_FILE'] = self.boto_path
      env['BOTO_CONFIG'] = self.boto_path
    else:
      custompath = env.get('AWS_CREDENTIAL_FILE', '~/.boto') + '.depot_tools'
      custompath = os.path.expanduser(custompath)
      if os.path.exists(custompath):
        env['AWS_CREDENTIAL_FILE'] = custompath

    return env

  def call(self, *args):
    cmd = [sys.executable, self.path]
    if self.bypass_prodaccess:
      cmd.append('--bypass_prodaccess')
    cmd.extend(args)
    return subprocess2.call(cmd, env=self.get_sub_env(), timeout=self.timeout)

  def check_call(self, *args):
    cmd = [sys.executable, self.path]
    if self.bypass_prodaccess:
      cmd.append('--bypass_prodaccess')
    cmd.extend(args)
    ((out, err), code) = subprocess2.communicate(
        cmd,
        stdout=subprocess2.PIPE,
        stderr=subprocess2.PIPE,
        env=self.get_sub_env(),
        timeout=self.timeout)

    # Parse output.
    status_code_match = re.search('status=([0-9]+)', err)
    if status_code_match:
      return (int(status_code_match.group(1)), out, err)
    if ('You are attempting to access protected data with '
          'no configured credentials.' in err):
      return (403, out, err)
    if 'No such object' in err:
      return (404, out, err)
    return (code, out, err)


def check_bucket_permissions(base_url, gsutil):
  code, _, ls_err = gsutil.check_call('ls', base_url)
  if code != 0:
    print >> sys.stderr, ls_err
  if code == 403:
    print >> sys.stderr, 'Got error 403 while authenticating to %s.' % base_url
    print >> sys.stderr, 'Try running "download_from_google_storage --config".'
  elif code == 404:
    print >> sys.stderr, '%s not found.' % base_url
  return (base_url, code)


def check_platform(target):
  """Checks if any parent directory of target matches (win|mac|linux)."""
  assert os.path.isabs(target)
  root, target_name = os.path.split(target)
  if not target_name:
    return None
  if target_name in ('linux', 'mac', 'win'):
    return target_name
  return check_platform(root)


def get_sha1(filename):
  sha1 = hashlib.sha1()
  with open(filename, 'rb') as f:
    while True:
      # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
      chunk = f.read(1024*1024)
      if not chunk:
        break
      sha1.update(chunk)
  return sha1.hexdigest()


# Download-specific code starts here

def enumerate_work_queue(input_filename, work_queue, directory,
                         recursive, ignore_errors, output, sha1_file,
                         auto_platform):
  if sha1_file:
    if not os.path.exists(input_filename):
      if not ignore_errors:
        raise FileNotFoundError('%s not found.' % input_filename)
      print >> sys.stderr, '%s not found.' % input_filename
    with open(input_filename, 'rb') as f:
      sha1_match = re.match('^([A-Za-z0-9]{40})$', f.read(1024).rstrip())
      if sha1_match:
        work_queue.put((sha1_match.groups(1)[0], output))
        return 1
    if not ignore_errors:
      raise InvalidFileError('No sha1 sum found in %s.' % input_filename)
    print >> sys.stderr, 'No sha1 sum found in %s.' % input_filename
    return 0

  if not directory:
    work_queue.put((input_filename, output))
    return 1

  work_queue_size = 0
  for root, dirs, files in os.walk(input_filename):
    if not recursive:
      for item in dirs[:]:
        dirs.remove(item)
    else:
      for exclude in ['.svn', '.git']:
        if exclude in dirs:
          dirs.remove(exclude)
    for filename in files:
      full_path = os.path.join(root, filename)
      if full_path.endswith('.sha1'):
        if auto_platform:
          # Skip if the platform does not match.
{'fraction_non_alphanumeric': 0.11643709825528008, 'fraction_numerical': 0.013406795224977043, 'mean_word_length': 3.14106463878327, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4016724', 'n_tokens_mistral': 3665, 'n_tokens_neox': 3512, 'n_words': 1000}