<commit_before>#!/usr/bin/env python
# coding=utf8
# Copyright 2013 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Main module for Google Cloud Storage command line tool."""

import ConfigParser
import errno
import getopt
import logging
import os
import re
import signal
import socket
import sys
import traceback

import boto.exception
from gslib import util
from gslib import wildcard_iterator
from gslib.command_runner import CommandRunner
import gslib.exception
import httplib2

# We don't use the oauth2 authentication plugin directly; importing it here
# ensures that it's loaded and available by default when an operation requiring
# authentication is performed.
try:
  from gslib.third_party.oauth2_plugin import oauth2_plugin
except ImportError:
  pass


GSLIB_DIR = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
GSUTIL_DIR = os.path.normpath(os.path.join(GSLIB_DIR, '..'))
debug = 0


def _OutputAndExit(message):
  if debug == 4:
    stack_trace = traceback.format_exc()
    err = ('DEBUG: Exception stack trace:\n    %s\n' %
           re.sub('\\n', '\n    ', stack_trace))
  else:
    err = '%s\n' % message
  sys.stderr.write(err.encode('utf-8'))
  sys.exit(1)


def _OutputUsageAndExit(command_runner):
  command_runner.RunNamedCommand('help')
  sys.exit(1)


def main():
  global debug

  if sys.version_info[:3] < (2, 6):
    raise gslib.exception.CommandException(
        'gsutil requires Python 2.6 or higher.')

  # Load the gsutil version number and append it to boto.UserAgent so the value
  # is set before anything instantiates boto. (If parts of boto were
  # instantiated first those parts would have the old value of boto.UserAgent,
  # so we wouldn't be guaranteed that all code paths send the correct user
  # agent.)
  boto.UserAgent += ' gsutil/%s (%s)' % (gslib.VERSION, sys.platform)

  config_file_list = _GetBotoConfigFileList()
  command_runner = CommandRunner(config_file_list)
  headers = {}
  parallel_operations = False
  quiet = False
  version = False
  debug = 0

  # If user enters no commands just print the usage info.
  if len(sys.argv) == 1:
    sys.argv.append('help')

  # Change the default of the 'https_validate_certificates' boto option to
  # True (it is currently False in boto).
  if not boto.config.has_option('Boto', 'https_validate_certificates'):
    if not boto.config.has_section('Boto'):
      boto.config.add_section('Boto')
    boto.config.setbool('Boto', 'https_validate_certificates', True)

  try:
    opts, args = getopt.getopt(sys.argv[1:], 'dDvh:mq',
                               ['debug', 'detailedDebug', 'version', 'help',
                                'header', 'multithreaded', 'quiet'])
  except getopt.GetoptError as e:
    _HandleCommandException(gslib.exception.CommandException(e.msg))
  for o, a in opts:
    if o in ('-d', '--debug'):
      # Passing debug=2 causes boto to include httplib header output.
      debug = 2
    elif o in ('-D', '--detailedDebug'):
      # We use debug level 3 to ask gsutil code to output more detailed
      # debug output. This is a bit of a hack since it overloads the same
      # flag that was originally implemented for boto use. And we use -DD
      # to ask for really detailed debugging (i.e., including HTTP payload).
      if debug == 3:
        debug = 4
      else:
        debug = 3
    elif o in ('-?', '--help'):
      _OutputUsageAndExit(command_runner)
    elif o in ('-h', '--header'):
      (hdr_name, unused_ptn, hdr_val) = a.partition(':')
      if not hdr_name:
        _OutputUsageAndExit(command_runner)
      headers[hdr_name] = hdr_val
    elif o in ('-m', '--multithreaded'):
      parallel_operations = True
    elif o in ('-q', '--quiet'):
      quiet = True
    elif o in ('-v', '--version'):
      version = True
  httplib2.debuglevel = debug
  if debug > 1:
    sys.stderr.write(
        '***************************** WARNING *****************************\n'
        '*** You are running gsutil with debug output enabled.\n'
        '*** Be aware that debug output includes authentication '
        'credentials.\n'
        '*** Do not share (e.g., post to support forums) debug output\n'
        '*** unless you have sanitized authentication tokens in the\n'
        '*** output, or have revoked your credentials.\n'
        '***************************** WARNING *****************************\n')
  if debug == 2:
    logging.basicConfig(level=logging.DEBUG)
  elif debug > 2:
    logging.basicConfig(level=logging.DEBUG)
    command_runner.RunNamedCommand('ver', ['-l'])
    config_items = []
    try:
      config_items.extend(boto.config.items('Boto'))
      config_items.extend(boto.config.items('GSUtil'))
    except ConfigParser.NoSectionError:
      pass
    sys.stderr.write('config_file_list: %s\n' % config_file_list)
    sys.stderr.write('config: %s\n' % str(config_items))
  elif quiet:
    logging.basicConfig(level=logging.WARNING)
  else:
    logging.basicConfig(level=logging.INFO)

  if version:
    command_name = 'version'
  elif not args:
    command_name = 'help'
  else:
    command_name = args[0]

  # Unset http_proxy environment variable if it's set, because it confuses
  # boto. (Proxies should instead be configured via the boto config file.)
  if 'http_proxy' in os.environ:
    if debug > 1:
      sys.stderr.write(
          'Unsetting http_proxy environment variable within gsutil run.\n')
    del os.environ['http_proxy']

  return _RunNamedCommandAndHandleExceptions(command_runner, command_name,
                                             args[1:], headers, debug,
                                             parallel_operations)
<commit_msg>Setting oauth2client's logging to behave more like gsutil's<commit_after>#!/usr/bin/env python
# coding=utf8
# Copyright 2013 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Main module for Google Cloud Storage command line tool."""

import ConfigParser
import errno
import getopt
import logging
import os
import re
import signal
import socket
import sys
import traceback

import boto.exception
from gslib import util
from gslib import wildcard_iterator
from gslib.command_runner import CommandRunner
import gslib.exception
import httplib2
import oauth2client

# We don't use the oauth2 authentication plugin directly; importing it here
# ensures that it's loaded and available by default when an operation requiring
# authentication is performed.
try:
  from gslib.third_party.oauth2_plugin import oauth2_plugin
except ImportError:
  pass


GSLIB_DIR = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
GSUTIL_DIR = os.path.normpath(os.path.join(GSLIB_DIR, '..'))
debug = 0


def _OutputAndExit(message):
  if debug == 4:
    stack_trace = traceback.format_exc()
    err = ('DEBUG: Exception stack trace:\n    %s\n' %
           re.sub('\\n', '\n    ', stack_trace))
  else:
    err = '%s\n' % message
  sys.stderr.write(err.encode('utf-8'))
  sys.exit(1)


def _OutputUsageAndExit(command_runner):
  command_runner.RunNamedCommand('help')
  sys.exit(1)


def main():
  global debug

  if sys.version_info[:3] < (2, 6):
    raise gslib.exception.CommandException(
        'gsutil requires Python 2.6 or higher.')

  # Load the gsutil version number and append it to boto.UserAgent so the value
  # is set before anything instantiates boto. (If parts of boto were
  # instantiated first those parts would have the old value of boto.UserAgent,
  # so we wouldn't be guaranteed that all code paths send the correct user
  # agent.)
  boto.UserAgent += ' gsutil/%s (%s)' % (gslib.VERSION, sys.platform)

  config_file_list = _GetBotoConfigFileList()
  command_runner = CommandRunner(config_file_list)
  headers = {}
  parallel_operations = False
  quiet = False
  version = False
  debug = 0

  # If user enters no commands just print the usage info.
  if len(sys.argv) == 1:
    sys.argv.append('help')

  # Change the default of the 'https_validate_certificates' boto option to
  # True (it is currently False in boto).
  if not boto.config.has_option('Boto', 'https_validate_certificates'):
    if not boto.config.has_section('Boto'):
      boto.config.add_section('Boto')
    boto.config.setbool('Boto', 'https_validate_certificates', True)

  try:
    opts, args = getopt.getopt(sys.argv[1:], 'dDvh:mq',
                               ['debug', 'detailedDebug', 'version', 'help',
                                'header', 'multithreaded', 'quiet'])
  except getopt.GetoptError as e:
    _HandleCommandException(gslib.exception.CommandException(e.msg))
  for o, a in opts:
    if o in ('-d', '--debug'):
      # Passing debug=2 causes boto to include httplib header output.
      debug = 2
    elif o in ('-D', '--detailedDebug'):
      # We use debug level 3 to ask gsutil code to output more detailed
      # debug output. This is a bit of a hack since it overloads the same
      # flag that was originally implemented for boto use. And we use -DD
      # to ask for really detailed debugging (i.e., including HTTP payload).
      if debug == 3:
        debug = 4
      else:
        debug = 3
    elif o in ('-?', '--help'):
      _OutputUsageAndExit(command_runner)
    elif o in ('-h', '--header'):
      (hdr_name, unused_ptn, hdr_val) = a.partition(':')
      if not hdr_name:
        _OutputUsageAndExit(command_runner)
      headers[hdr_name] = hdr_val
    elif o in ('-m', '--multithreaded'):
      parallel_operations = True
    elif o in ('-q', '--quiet'):
      quiet = True
    elif o in ('-v', '--version'):
      version = True
  httplib2.debuglevel = debug
  if debug > 1:
    sys.stderr.write(
        '***************************** WARNING *****************************\n'
        '*** You are running gsutil with debug output enabled.\n'
        '*** Be aware that debug output includes authentication '
        'credentials.\n'
        '*** Do not share (e.g., post to support forums) debug output\n'
        '*** unless you have sanitized authentication tokens in the\n'
        '*** output, or have revoked your credentials.\n'
        '***************************** WARNING *****************************\n')
  if debug == 2:
    logging.basicConfig(level=logging.DEBUG)
    oauth2client.client.logger.setLevel(logging.DEBUG)
  elif debug > 2:
    logging.basicConfig(level=logging.DEBUG)
    oauth2client.client.logger.setLevel(logging.DEBUG)
    command_runner.RunNamedCommand('ver', ['-l'])
    config_items = []
    try:
      config_items.extend(boto.config.items('Boto'))
      config_items.extend(boto.config.items('GSUtil'))
    except ConfigParser.NoSectionError:
      pass
    sys.stderr.write('config_file_list: %s\n' % config_file_list)
    sys.stderr.write('config: %s\n' % str(config_items))
  elif quiet:
    logging.basicConfig(level=logging.WARNING)
    oauth2client.client.logger.setLevel(logging.WARNING)
  else:
    logging.basicConfig(level=logging.INFO)
    # oauth2client uses info logging in places that would better correspond to
    # gsutil's debug logging (e.g., when refreshing access tokens).
    oauth2client.client.logger.setLevel(logging.WARNING)

  if version:
    command_name = 'version'
  elif not args:
    command_name = 'help'
  else:
    command_name = args[0]

  # Unset http_proxy environment variable if it's set, because it confuses
  # boto. (Proxies should instead be configured via the boto config file.)
  if 'http_proxy' in os.environ:
    if debug > 1:
      sys.stderr.write(
          'Unsetting http_proxy environment variable within gsutil run.\n')
    del os.environ['http_proxy']

  return _RunNamedCommandAndHandleExceptions(command_runner, command_name,
                                             args[1:], headers, debug,
                                             parallel_operations)

{'fraction_non_alphanumeric': 0.1203060348192677, 'fraction_numerical': 0.006635959091263955, 'mean_word_length': 3.2082785808147176, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5677324', 'n_tokens_mistral': 3879, 'n_tokens_neox': 3780, 'n_words': 1234}