<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Get stats about your activity.

Example:
  - my_activity.py  for stats for the current week (last week on mondays).
  - my_activity.py -Q  for stats for last quarter.
  - my_activity.py -Y  for stats for this year.
  - my_activity.py -b 4/5/12  for stats since 4/5/12.
  - my_activity.py -b 4/5/12 -e 6/7/12 for stats between 4/5/12 and 6/7/12.
"""

# These services typically only provide a created time and a last modified time
# for each item for general queries. This is not enough to determine if there
# was activity in a given time period. So, we first query for all things created
# before end and modified after begin. Then, we get the details of each item and
# check those details to determine if there was activity in the given period.
# This means that query time scales mostly with (today() - begin).

import cookielib
import datetime
from datetime import datetime
from datetime import timedelta
from functools import partial
import json
import optparse
import os
import subprocess
import sys
import urllib
import urllib2

import gerrit_util
import rietveld
from third_party import upload

try:
  from dateutil.relativedelta import relativedelta # pylint: disable=F0401
except ImportError:
  print 'python-dateutil package required'
  exit(1)

# python-keyring provides easy access to the system keyring.
try:
  import keyring  # pylint: disable=W0611,F0401
except ImportError:
  print 'Consider installing python-keyring'

rietveld_instances = [
  {
    'url': 'codereview.chromium.org',
    'shorturl': 'crrev.com',
    'supports_owner_modified_query': True,
    'requires_auth': False,
    'email_domain': 'chromium.org',
  },
  {
    'url': 'chromereviews.googleplex.com',
    'shorturl': 'go/chromerev',
    'supports_owner_modified_query': True,
    'requires_auth': True,
    'email_domain': 'google.com',
  },
  {
    'url': 'codereview.appspot.com',
    'supports_owner_modified_query': True,
    'requires_auth': False,
    'email_domain': 'chromium.org',
  },
  {
    'url': 'breakpad.appspot.com',
    'supports_owner_modified_query': False,
    'requires_auth': False,
    'email_domain': 'chromium.org',
  },
]

gerrit_instances = [
  {
    'url': 'chromium-review.googlesource.com',
    'shorturl': 'crosreview.com',
  },
  {
    'url': 'chrome-internal-review.googlesource.com',
    'shorturl': 'crosreview.com/i',
  },
  {
    'host': 'gerrit.chromium.org',
    'port': 29418,
  },
  {
    'host': 'gerrit-int.chromium.org',
    'port': 29419,
  },
]

google_code_projects = [
  {
    'name': 'chromium',
    'shorturl': 'crbug.com',
  },
  {
    'name': 'chromium-os',
    'shorturl': 'crosbug.com',
  },
  {
    'name': 'chrome-os-partner',
  },
  {
    'name': 'google-breakpad',
  },
  {
    'name': 'gyp',
  },
  {
    'name': 'skia',
  },
]

# Uses ClientLogin to authenticate the user for Google Code issue trackers.
def get_auth_token(email):
  # KeyringCreds will use the system keyring on the first try, and prompt for
  # a password on the next ones.
  creds = upload.KeyringCreds('code.google.com', 'code.google.com', email)
  for _ in xrange(3):
    email, password = creds.GetUserCredentials()
    url = 'https://www.google.com/accounts/ClientLogin'
    data = urllib.urlencode({
        'Email': email,
        'Passwd': password,
        'service': 'code',
        'source': 'chrome-my-activity',
        'accountType': 'GOOGLE',
    })
    req = urllib2.Request(url, data=data, headers={'Accept': 'text/plain'})
    try:
      response = urllib2.urlopen(req)
      response_body = response.read()
      response_dict = dict(x.split('=')
                           for x in response_body.split('\n') if x)
      return response_dict['Auth']
    except urllib2.HTTPError, e:
      print e

  print 'Unable to authenticate to code.google.com.'
  print 'Some issues may be missing.'
  return None


def username(email):
  """Keeps the username of an email address."""
  return email and email.split('@', 1)[0]


def datetime_to_midnight(date):
  return date - timedelta(hours=date.hour, minutes=date.minute,
                          seconds=date.second, microseconds=date.microsecond)


def get_quarter_of(date):
  begin = (datetime_to_midnight(date) -
           relativedelta(months=(date.month % 3) - 1, days=(date.day - 1)))
  return begin, begin + relativedelta(months=3)


def get_year_of(date):
  begin = (datetime_to_midnight(date) -
           relativedelta(months=(date.month - 1), days=(date.day - 1)))
  return begin, begin + relativedelta(years=1)


def get_week_of(date):
  begin = (datetime_to_midnight(date) - timedelta(days=date.weekday()))
  return begin, begin + timedelta(days=7)


def get_yes_or_no(msg):
  while True:
    response = raw_input(msg + ' yes/no [no] ')
    if response == 'y' or response == 'yes':
      return True
    elif not response or response == 'n' or response == 'no':
      return False


def datetime_from_gerrit(date_string):
  return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f000')


def datetime_from_rietveld(date_string):
  return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f')


def datetime_from_google_code(date_string):
  return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S.%fZ')


class MyActivity(object):
  def __init__(self, options):
    self.options = options
    self.modified_after = options.begin
    self.modified_before = options.end
    self.user = options.user
    self.changes = []
    self.reviews = []
    self.issues = []
    self.check_cookies()
    self.google_code_auth_token = None

  # Check the codereview cookie jar to determine which Rietveld instances to
  # authenticate to.
  def check_cookies(self):
    cookie_file = os.path.expanduser('~/.codereview_upload_cookies')
    if not os.path.exists(cookie_file):
      print 'No Rietveld cookie file found.'
      cookie_jar = []
    else:<commit_msg>my_activity.py: Parse rietveld dates without milliseconds.<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Get stats about your activity.

Example:
  - my_activity.py  for stats for the current week (last week on mondays).
  - my_activity.py -Q  for stats for last quarter.
  - my_activity.py -Y  for stats for this year.
  - my_activity.py -b 4/5/12  for stats since 4/5/12.
  - my_activity.py -b 4/5/12 -e 6/7/12 for stats between 4/5/12 and 6/7/12.
"""

# These services typically only provide a created time and a last modified time
# for each item for general queries. This is not enough to determine if there
# was activity in a given time period. So, we first query for all things created
# before end and modified after begin. Then, we get the details of each item and
# check those details to determine if there was activity in the given period.
# This means that query time scales mostly with (today() - begin).

import cookielib
import datetime
from datetime import datetime
from datetime import timedelta
from functools import partial
import json
import optparse
import os
import subprocess
import sys
import urllib
import urllib2

import gerrit_util
import rietveld
from third_party import upload

try:
  from dateutil.relativedelta import relativedelta # pylint: disable=F0401
except ImportError:
  print 'python-dateutil package required'
  exit(1)

# python-keyring provides easy access to the system keyring.
try:
  import keyring  # pylint: disable=W0611,F0401
except ImportError:
  print 'Consider installing python-keyring'

rietveld_instances = [
  {
    'url': 'codereview.chromium.org',
    'shorturl': 'crrev.com',
    'supports_owner_modified_query': True,
    'requires_auth': False,
    'email_domain': 'chromium.org',
  },
  {
    'url': 'chromereviews.googleplex.com',
    'shorturl': 'go/chromerev',
    'supports_owner_modified_query': True,
    'requires_auth': True,
    'email_domain': 'google.com',
  },
  {
    'url': 'codereview.appspot.com',
    'supports_owner_modified_query': True,
    'requires_auth': False,
    'email_domain': 'chromium.org',
  },
  {
    'url': 'breakpad.appspot.com',
    'supports_owner_modified_query': False,
    'requires_auth': False,
    'email_domain': 'chromium.org',
  },
]

gerrit_instances = [
  {
    'url': 'chromium-review.googlesource.com',
    'shorturl': 'crosreview.com',
  },
  {
    'url': 'chrome-internal-review.googlesource.com',
    'shorturl': 'crosreview.com/i',
  },
  {
    'host': 'gerrit.chromium.org',
    'port': 29418,
  },
  {
    'host': 'gerrit-int.chromium.org',
    'port': 29419,
  },
]

google_code_projects = [
  {
    'name': 'chromium',
    'shorturl': 'crbug.com',
  },
  {
    'name': 'chromium-os',
    'shorturl': 'crosbug.com',
  },
  {
    'name': 'chrome-os-partner',
  },
  {
    'name': 'google-breakpad',
  },
  {
    'name': 'gyp',
  },
  {
    'name': 'skia',
  },
]

# Uses ClientLogin to authenticate the user for Google Code issue trackers.
def get_auth_token(email):
  # KeyringCreds will use the system keyring on the first try, and prompt for
  # a password on the next ones.
  creds = upload.KeyringCreds('code.google.com', 'code.google.com', email)
  for _ in xrange(3):
    email, password = creds.GetUserCredentials()
    url = 'https://www.google.com/accounts/ClientLogin'
    data = urllib.urlencode({
        'Email': email,
        'Passwd': password,
        'service': 'code',
        'source': 'chrome-my-activity',
        'accountType': 'GOOGLE',
    })
    req = urllib2.Request(url, data=data, headers={'Accept': 'text/plain'})
    try:
      response = urllib2.urlopen(req)
      response_body = response.read()
      response_dict = dict(x.split('=')
                           for x in response_body.split('\n') if x)
      return response_dict['Auth']
    except urllib2.HTTPError, e:
      print e

  print 'Unable to authenticate to code.google.com.'
  print 'Some issues may be missing.'
  return None


def username(email):
  """Keeps the username of an email address."""
  return email and email.split('@', 1)[0]


def datetime_to_midnight(date):
  return date - timedelta(hours=date.hour, minutes=date.minute,
                          seconds=date.second, microseconds=date.microsecond)


def get_quarter_of(date):
  begin = (datetime_to_midnight(date) -
           relativedelta(months=(date.month % 3) - 1, days=(date.day - 1)))
  return begin, begin + relativedelta(months=3)


def get_year_of(date):
  begin = (datetime_to_midnight(date) -
           relativedelta(months=(date.month - 1), days=(date.day - 1)))
  return begin, begin + relativedelta(years=1)


def get_week_of(date):
  begin = (datetime_to_midnight(date) - timedelta(days=date.weekday()))
  return begin, begin + timedelta(days=7)


def get_yes_or_no(msg):
  while True:
    response = raw_input(msg + ' yes/no [no] ')
    if response == 'y' or response == 'yes':
      return True
    elif not response or response == 'n' or response == 'no':
      return False


def datetime_from_gerrit(date_string):
  return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f000')


def datetime_from_rietveld(date_string):
  try:
    return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f')
  except ValueError:
    # Sometimes rietveld returns a value without the milliseconds part, so we
    # attempt to parse those cases as well.
    return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S')


def datetime_from_google_code(date_string):
  return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S.%fZ')


class MyActivity(object):
  def __init__(self, options):
    self.options = options
    self.modified_after = options.begin
    self.modified_before = options.end
    self.user = options.user
    self.changes = []
    self.reviews = []
    self.issues = []
    self.check_cookies()
    self.google_code_auth_token = None

  # Check the codereview cookie jar to determine which Rietveld instances to
  # authenticate to.
  def check_cookies(self):
    cookie_file = os.path.expanduser('~/.codereview_upload_cookies')
    if not os.path.exists(cookie_file):
      print 'No Rietveld cookie file found.'
      cookie_jar = []
    else:
{'fraction_non_alphanumeric': 0.1261720012932428, 'fraction_numerical': 0.011154219204655674, 'mean_word_length': 3.4173509460906817, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1594661', 'n_tokens_mistral': 4300, 'n_tokens_neox': 4118, 'n_words': 1115}