<commit_before>from googleapiclient import discovery
from googleapiclient import errors
from multiprocessing import pool
from oauth2client import client
from oauth2client import keyring_storage
from oauth2client import tools
from protorpc import messages
from protorpc import message_types
from protorpc import protojson
import base64
import httplib2
import json
import logging
import md5
import mimetypes
import os
import progressbar
import requests
import threading

# Google API details for a native/installed application for API project grow-prod.
CLIENT_ID = '578372381550-jfl3hdlf1q5rgib94pqsctv1kgkflu1a.apps.googleusercontent.com'
CLIENT_SECRET = '<KEY>'  # Not so secret for installed apps.
REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'
OAUTH_SCOPES = [
    'https://www.googleapis.com/auth/plus.me',
    'https://www.googleapis.com/auth/userinfo.email',
]

requests_logger = logging.getLogger('requests')
requests_logger.setLevel(logging.WARNING)


class Error(Exception):
  pass


class Verb(object):
  GET = 'GET'
  PUT = 'PUT'
  DELETE = 'DELETE'


class AuthorMessage(messages.Message):
  name = messages.StringField(1)
  email = messages.StringField(2)


class CommitMessage(messages.Message):
  sha = messages.StringField(1)
  author = messages.MessageField(AuthorMessage, 2)
  date = message_types.DateTimeField(3)
  message = messages.StringField(4)
  has_unstaged_changes = messages.BooleanField(5)
  branch = messages.StringField(6)


class HttpWithApiKey(httplib2.Http):

  def __init__(self, *args, **kwargs):
    self.api_key = kwargs.pop('api_key', None)
    super(HttpWithApiKey, self).__init__(*args, **kwargs)

  def _request(self, conn, host, absolute_uri, request_uri, method, body, headers,
               redirections, cachekey):
    if headers is None:
      headers = {}
    if self.api_key is not None:
      headers['WebReview-Api-Key'] = self.api_key
    return super(HttpWithApiKey, self)._request(
        conn, host, absolute_uri, request_uri, method, body, headers,
        redirections, cachekey)


class RpcError(Error):

  def __init__(self, status, message=None, data=None):
    self.status = status
    self.message = data['error_message'] if data else message
    self.data = data

  def __repr__(self):
    return '{}: {}'.format(self.status, self.message)

  def __str__(self):
    return '{}: {}'.format(self.status, self.message)

  def __getitem__(self, name):
    return self.data[name]


class WebReviewRpcError(RpcError):
  pass


class GoogleStorageRpcError(RpcError, IOError):
  pass


class WebReview(object):
  _pool_size = 10

  def __init__(self, project=None, name=None, commit=None, host=None,
               secure=False, username='default', api='webreview',
               version='v0', api_key=None):
    if '/' not in project:
      raise ValueError('Project must be in format: <owner>/<project>')
    self.owner, self.project = project.split('/')
    self.name = name
    self.gs = GoogleStorageSigner()
    self.lock = threading.Lock()
    self.pool = pool.ThreadPool(processes=self._pool_size)
    self.commit = commit
    root = '{}://{}/_ah/api'.format('https' if secure else 'http', host)
    self.api_key = api_key
    self._api = api
    self._version = version
    self._url = '{}/discovery/v1/apis/{}/{}/rest'.format(root, api, version)
    self._service = None

  @property
  def fileset(self):
    commit = None
    if self.commit:
      commit = json.loads(protojson.encode_message(self.commit))
    return {
        'name': self.name,
        'commit': commit,
        'project': {
            'nickname': self.project,
            'owner': {
                'nickname': self.owner,
            },
        },
    }

  def get_service(self, username='default', reauth=False):
    http = HttpWithApiKey(api_key=self.api_key)
    if self.api_key is None:
      credentials = WebReview.get_credentials(username=username, reauth=reauth)
      credentials.authorize(http)
      if credentials.access_token_expired:
        credentials.refresh(http)
    return discovery.build(
        self._api,
        self._version,
        discoveryServiceUrl=self._url,
        http=http)

  def login(self, username='default', reauth=False):
    self._service = self.get_service(username=username, reauth=reauth)

  @property
  def service(self):
    if self._service is not None:
      return self._service
    self._service = self.get_service()
    return self._service

  @staticmethod
  def get_credentials(username, reauth=False):
    storage = keyring_storage.Storage('WebReview Client', username)
    credentials = storage.get()
    if credentials and not credentials.invalid:
      return credentials
    if credentials is None or reauth:
      parser = tools.argparser
      flags, _ = parser.parse_known_args([])
      flow = client.OAuth2WebServerFlow(CLIENT_ID, CLIENT_SECRET, OAUTH_SCOPES,
                                        redirect_uri=REDIRECT_URI)
      credentials = tools.run_flow(flow, storage, flags)
    return credentials

  def upload_dir(self, build_dir):
    paths_to_contents = WebReview._get_paths_to_contents_from_dir(build_dir)
    return self.write(paths_to_contents)

  def delete(self, paths):
    paths_to_contents = dict([(path, None) for path in paths])
    req = self.gs.create_sign_requests_request(Verb.DELETE, self.fileset, paths_to_contents)
    try:
      resp = self.service.sign_requests(body=req).execute()
    except errors.HttpError as e:
      raise WebReviewRpcError(e.resp.status, e._get_reason().strip())
    return self._execute_signed_requests(resp['signed_requests'], paths_to_contents)

  def read(self, paths):
    paths_to_contents = dict([(path, None) for path in paths])
    req = self.gs.create_sign_requests_request(Verb.GET, self.fileset, paths_to_contents)
    try:
      resp = self.service.sign_requests(body=req).execute()<commit_msg>Add support for 'INTERACTIVE_AUTH' env variable.<commit_after>from googleapiclient import discovery
from googleapiclient import errors
from multiprocessing import pool
from oauth2client import client
from oauth2client import keyring_storage
from oauth2client import tools
from protorpc import messages
from protorpc import message_types
from protorpc import protojson
import base64
import httplib2
import json
import logging
import md5
import mimetypes
import os
import progressbar
import requests
import threading

# Google API details for a native/installed application for API project grow-prod.
CLIENT_ID = '578372381550-jfl3hdlf1q5rgib94pqsctv1kgkflu1a.apps.googleusercontent.com'
CLIENT_SECRET = 'XQKqbwTg88XVpaBNRcm_tYLf'  # Not so secret for installed apps.
REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'
OAUTH_SCOPES = [
    'https://www.googleapis.com/auth/plus.me',
    'https://www.googleapis.com/auth/userinfo.email',
]

requests_logger = logging.getLogger('requests')
requests_logger.setLevel(logging.WARNING)


class Error(Exception):
  pass


class Verb(object):
  GET = 'GET'
  PUT = 'PUT'
  DELETE = 'DELETE'


class AuthorMessage(messages.Message):
  name = messages.StringField(1)
  email = messages.StringField(2)


class CommitMessage(messages.Message):
  sha = messages.StringField(1)
  author = messages.MessageField(AuthorMessage, 2)
  date = message_types.DateTimeField(3)
  message = messages.StringField(4)
  has_unstaged_changes = messages.BooleanField(5)
  branch = messages.StringField(6)


class HttpWithApiKey(httplib2.Http):

  def __init__(self, *args, **kwargs):
    self.api_key = kwargs.pop('api_key', None)
    super(HttpWithApiKey, self).__init__(*args, **kwargs)

  def _request(self, conn, host, absolute_uri, request_uri, method, body, headers,
               redirections, cachekey):
    if headers is None:
      headers = {}
    if self.api_key is not None:
      headers['WebReview-Api-Key'] = self.api_key
    return super(HttpWithApiKey, self)._request(
        conn, host, absolute_uri, request_uri, method, body, headers,
        redirections, cachekey)


class RpcError(Error):

  def __init__(self, status, message=None, data=None):
    self.status = status
    self.message = data['error_message'] if data else message
    self.data = data

  def __repr__(self):
    return '{}: {}'.format(self.status, self.message)

  def __str__(self):
    return '{}: {}'.format(self.status, self.message)

  def __getitem__(self, name):
    return self.data[name]


class WebReviewRpcError(RpcError):
  pass


class GoogleStorageRpcError(RpcError, IOError):
  pass


class WebReview(object):
  _pool_size = 10

  def __init__(self, project=None, name=None, commit=None, host=None,
               secure=False, username='default', api='webreview',
               version='v0', api_key=None):
    if '/' not in project:
      raise ValueError('Project must be in format: <owner>/<project>')
    self.owner, self.project = project.split('/')
    self.name = name
    self.gs = GoogleStorageSigner()
    self.lock = threading.Lock()
    self.pool = pool.ThreadPool(processes=self._pool_size)
    self.commit = commit
    root = '{}://{}/_ah/api'.format('https' if secure else 'http', host)
    self.api_key = api_key
    self._api = api
    self._version = version
    self._url = '{}/discovery/v1/apis/{}/{}/rest'.format(root, api, version)
    self._service = None

  @property
  def fileset(self):
    commit = None
    if self.commit:
      commit = json.loads(protojson.encode_message(self.commit))
    return {
        'name': self.name,
        'commit': commit,
        'project': {
            'nickname': self.project,
            'owner': {
                'nickname': self.owner,
            },
        },
    }

  def get_service(self, username='default', reauth=False):
    http = HttpWithApiKey(api_key=self.api_key)
    if self.api_key is None:
      credentials = WebReview.get_credentials(username=username, reauth=reauth)
      credentials.authorize(http)
      if credentials.access_token_expired:
        credentials.refresh(http)
    return discovery.build(
        self._api,
        self._version,
        discoveryServiceUrl=self._url,
        http=http)

  def login(self, username='default', reauth=False):
    self._service = self.get_service(username=username, reauth=reauth)

  @property
  def service(self):
    if self._service is not None:
      return self._service
    self._service = self.get_service()
    return self._service

  @staticmethod
  def get_credentials(username, reauth=False):
    storage = keyring_storage.Storage('WebReview Client', username)
    credentials = storage.get()
    if credentials and not credentials.invalid:
      return credentials
    if credentials is None or reauth:
      parser = tools.argparser
      if os.getenv('INTERACTIVE_AUTH'):
        args = []
      else:
        args = ['--noauth_local_webserver']
      flags, _ = parser.parse_known_args(args)
      flow = client.OAuth2WebServerFlow(CLIENT_ID, CLIENT_SECRET, OAUTH_SCOPES,
                                        redirect_uri=REDIRECT_URI)
      credentials = tools.run_flow(flow, storage, flags)
    return credentials

  def upload_dir(self, build_dir):
    paths_to_contents = WebReview._get_paths_to_contents_from_dir(build_dir)
    return self.write(paths_to_contents)

  def delete(self, paths):
    paths_to_contents = dict([(path, None) for path in paths])
    req = self.gs.create_sign_requests_request(Verb.DELETE, self.fileset, paths_to_contents)
    try:
      resp = self.service.sign_requests(body=req).execute()
    except errors.HttpError as e:
      raise WebReviewRpcError(e.resp.status, e._get_reason().strip())
    return self._execute_signed_requests(resp['signed_requests'], paths_to_contents)

  def read(self, paths):
    paths_to_contents = dict([(path, None) for path in paths])
    req = self.gs.create_sign_requests_request(Verb.GET, self.fileset, paths_to_contents)
    try:
      resp = self.service.sign_requests(body=req).execute()
{'fraction_non_alphanumeric': 0.10873949579831933, 'fraction_numerical': 0.007226890756302521, 'mean_word_length': 3.6652293218345746, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2450106', 'n_tokens_mistral': 3777, 'n_tokens_neox': 3708, 'n_words': 838}