<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import datetime
import hashlib
import json
import os
import pytz
import time
import urllib
import urlparse

from buildbot.changes import base
from buildbot.schedulers.trysched import BadJobfile
from buildbot.status.builder import EXCEPTION
from twisted.application import internet
from twisted.internet import defer
from twisted.python import log
from twisted.web import client

from master import master_utils
from master.try_job_base import TryJobBase


# Number of recent buildsets used to initialize RietveldPollerWithCache's cache.
MAX_RECENT_BUILDSETS_TO_INIT_CACHE = 10000


def str_to_datetime(text):
  try:
    return datetime.datetime.strptime(text, '%Y-%m-%d %H:%M:%S.%f')
  except ValueError:
    return datetime.datetime.strptime(text, '%Y-%m-%d %H:%M:%S')


class _ValidUserPoller(internet.TimerService):
  """Check chromium-access for users allowed to send jobs from Rietveld.
  """
  # The name of the file that contains the password for authenticating
  # requests to chromium-access.
  _PWD_FILE = '.try_job_rietveld_password'
  _NORMAL_DOMAIN = '@chromium.org'
  _SPECIAL_DOMAIN = '@google.com'

  def __init__(self, interval):
    """
    Args:
      interval: Interval used to poll chromium-access, in seconds.
    """
    if interval:
      internet.TimerService.__init__(self, interval, _ValidUserPoller._poll,
                                     self)
    self._users = frozenset()
    self._valid = False

  def contains(self, email):
    """Checks if the given email address is a valid user.

    Args:
      email: The email address to check against the internal list.

    Returns:
      True if the email address is allowed to send try jobs from rietveld.
    """
    if email is None:
      return False
    return email in self._users or email.endswith(self._SPECIAL_DOMAIN)

  def valid(self):
    """Returns true if the poller has retrieved valid users successfully."""
    return self._valid

  # base.PollingChangeSource overrides:
  def _poll(self):
    """Polls for valid user names.

    Returns:
      A deferred objects to be called once the operation completes.
    """
    log.msg('ValidUserPoller._poll')
    d = defer.succeed(None)
    d.addCallback(self._GetUsers)
    d.addCallback(self._MakeSet)
    d.addErrback(log.err, 'error in ValidUserPoller')
    return d

  def _GetUsers(self, _):
    """Downloads list of valid users.

    Returns:
      A string of lines containing the email addresses of users allowed to
      send jobs from Rietveld.
    """
    if not os.path.isfile(self._PWD_FILE):
      log.msg("No password file '%s'; no valid users." % self._PWD_FILE)
      return ""

    pwd = open(self._PWD_FILE).readline().strip()
    now_string = str(int(time.time()))
    params = {
      'md5': hashlib.md5(pwd + now_string).hexdigest(),
      'time': now_string
    }

    return client.getPage('https://chromium-access.appspot.com/auto/all_users',
                          agent='buildbot',
                          method='POST',
                          postdata=urllib.urlencode(params))

  def _MakeSet(self, data):
    """Converts the input data string into a set of email addresses.
    """
    emails = (email.strip() for email in data.splitlines())
    self._users = frozenset(email for email in emails if email)
    log.msg('Found %d users' % len(self._users))
    self._valid = True


class _RietveldPollerWithCache(base.PollingChangeSource):
  """Polls Rietveld for any pending patch sets to build.

  Periodically polls Rietveld to see if any patch sets have been marked by
  users to be tried. If so, send them to the trybots. Uses cursor to download
  all pages within a single poll. To avoid sending same jobs, keeps a cache of
  the jobs that were already processed thus reducing chances of a duplicate job
  submission and increasing robustness to bugs in Rietveld. On the first poll
  this cache is initialized with jobs currently pending on the Buildbot.
  """

  def __init__(self, pending_jobs_url, interval):
    """
    Args:
      pending_jobs_url: Rietveld URL string used to retrieve jobs to try.
      interval: Interval used to poll Rietveld, in seconds.
    """
    self.pollInterval = interval
    self._try_job_rietveld = None
    self._pending_jobs_url = pending_jobs_url
    self._processed_keys = None

  # base.PollingChangeSource overrides:<commit_msg>Re-add _RietveldPoller to try_job_rietveld module<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import datetime
import hashlib
import json
import os
import pytz
import time
import urllib
import urlparse

from buildbot.changes import base
from buildbot.schedulers.trysched import BadJobfile
from buildbot.status.builder import EXCEPTION
from twisted.application import internet
from twisted.internet import defer
from twisted.python import log
from twisted.web import client

from master import master_utils
from master.try_job_base import TryJobBase


# Number of recent buildsets used to initialize RietveldPollerWithCache's cache.
MAX_RECENT_BUILDSETS_TO_INIT_CACHE = 10000


def str_to_datetime(text):
  try:
    return datetime.datetime.strptime(text, '%Y-%m-%d %H:%M:%S.%f')
  except ValueError:
    return datetime.datetime.strptime(text, '%Y-%m-%d %H:%M:%S')


class _ValidUserPoller(internet.TimerService):
  """Check chromium-access for users allowed to send jobs from Rietveld.
  """
  # The name of the file that contains the password for authenticating
  # requests to chromium-access.
  _PWD_FILE = '.try_job_rietveld_password'
  _NORMAL_DOMAIN = '@chromium.org'
  _SPECIAL_DOMAIN = '@google.com'

  def __init__(self, interval):
    """
    Args:
      interval: Interval used to poll chromium-access, in seconds.
    """
    if interval:
      internet.TimerService.__init__(self, interval, _ValidUserPoller._poll,
                                     self)
    self._users = frozenset()
    self._valid = False

  def contains(self, email):
    """Checks if the given email address is a valid user.

    Args:
      email: The email address to check against the internal list.

    Returns:
      True if the email address is allowed to send try jobs from rietveld.
    """
    if email is None:
      return False
    return email in self._users or email.endswith(self._SPECIAL_DOMAIN)

  def valid(self):
    """Returns true if the poller has retrieved valid users successfully."""
    return self._valid

  # base.PollingChangeSource overrides:
  def _poll(self):
    """Polls for valid user names.

    Returns:
      A deferred objects to be called once the operation completes.
    """
    log.msg('ValidUserPoller._poll')
    d = defer.succeed(None)
    d.addCallback(self._GetUsers)
    d.addCallback(self._MakeSet)
    d.addErrback(log.err, 'error in ValidUserPoller')
    return d

  def _GetUsers(self, _):
    """Downloads list of valid users.

    Returns:
      A string of lines containing the email addresses of users allowed to
      send jobs from Rietveld.
    """
    if not os.path.isfile(self._PWD_FILE):
      log.msg("No password file '%s'; no valid users." % self._PWD_FILE)
      return ""

    pwd = open(self._PWD_FILE).readline().strip()
    now_string = str(int(time.time()))
    params = {
      'md5': hashlib.md5(pwd + now_string).hexdigest(),
      'time': now_string
    }

    return client.getPage('https://chromium-access.appspot.com/auto/all_users',
                          agent='buildbot',
                          method='POST',
                          postdata=urllib.urlencode(params))

  def _MakeSet(self, data):
    """Converts the input data string into a set of email addresses.
    """
    emails = (email.strip() for email in data.splitlines())
    self._users = frozenset(email for email in emails if email)
    log.msg('Found %d users' % len(self._users))
    self._valid = True


class _RietveldPoller(base.PollingChangeSource):
  """Polls Rietveld for any pending patch sets to build.

  Periodically polls Rietveld to see if any patch sets have been marked by
  users to be tried.  If so, send them to the trybots.
  """

  def __init__(self, get_pending_endpoint, interval, cachepath=None):
    """
    Args:
      get_pending_endpoint: Rietveld URL string used to retrieve jobs to try.
      interval: Interval used to poll Rietveld, in seconds.
      cachepath: Path to file where state to persist between master restarts
                 will be stored.
    """
    # Set interval time in base class.
    self.pollInterval = interval

    # A string URL for the Rietveld endpoint to query for pending try jobs.
    self._get_pending_endpoint = get_pending_endpoint

    # Cursor used to keep track of next patchset(s) to try.  If the cursor
    # is None, then try from the beginning.
    self._cursor = None

    # Try job parent of this poller.
    self._try_job_rietveld = None

    self._cachepath = cachepath

    if self._cachepath:
      if os.path.exists(self._cachepath):
        with open(self._cachepath) as f:
          # Using JSON allows us to be flexible and extend the format
          # in a compatible way.
          data = json.load(f)
          self._cursor = data.get('cursor').encode('utf-8')

  # base.PollingChangeSource overrides:
  def poll(self):
    """Polls Rietveld for any pending try jobs and submit them.

    Returns:
      A deferred objects to be called once the operation completes.
    """
    log.msg('RietveldPoller.poll')
    d = defer.succeed(None)
    d.addCallback(self._OpenUrl)
    d.addCallback(self._ParseJson)
    d.addErrback(log.err, 'error in RietveldPoller')  # eat errors
    return d

  def setServiceParent(self, parent):
    base.PollingChangeSource.setServiceParent(self, parent)
    self._try_job_rietveld = parent

  def _OpenUrl(self, _):
    """Downloads pending patch sets from Rietveld.

    Returns: A string containing the pending patchsets from Rietveld
        encoded as JSON.
    """
    endpoint = self._get_pending_endpoint
    if self._cursor:
      sep = '&' if '?' in endpoint else '?'
      endpoint = endpoint + '%scursor=%s' % (sep, self._cursor)

    log.msg('RietveldPoller._OpenUrl: %s' % endpoint)
    return client.getPage(endpoint, agent='buildbot', timeout=2*60)

  def _ParseJson(self, json_string):
    """Parses the JSON pending patch set information.

    Args:
      json_string: A string containing the serialized JSON jobs.

    Returns: A list of pending try jobs.  This is the list of all jobs returned
        by Rietveld, not simply the ones we tried this time.
    """
    data = json.loads(json_string)
    d = self._try_job_rietveld.SubmitJobs(data['jobs'])
    def success_callback(value):
      self._cursor = str(data['cursor'])
      self._try_job_rietveld.processed_keys.clear()

      if self._cachepath:
        with open(self._cachepath, 'w') as f:
          json.dump({'cursor': self._cursor}, f)

      return value
    d.addCallback(success_callback)
    return d


class _RietveldPollerWithCache(base.PollingChangeSource):
  """Polls Rietveld for any pending patch sets to build.

  Periodically polls Rietveld to see if any patch sets have been marked by
  users to be tried. If so, send them to the trybots. Uses cursor to download
  all pages within a single poll. To avoid sending same jobs, keeps a cache of
  the jobs that were already processed thus reducing chances of a duplicate job
  submission and increasing robustness to bugs in Rietveld. On the first poll
  this cache is initialized with jobs currently pending on the Buildbot.
  """

  def __init__(self, pending_jobs_url, interval):
    """
    Args:
      pending_jobs_url: Rietveld URL string used to retrieve jobs to try.
      interval: Interval used to poll Rietveld, in seconds.
    """
    self.pollInterval = interval
    self._try_job_rietveld = None
    self._pending_jobs_url = pending_jobs_url
    self._processed_keys = None

  # base.PollingChangeSource overrides:
{'fraction_non_alphanumeric': 0.08372931345240046, 'fraction_numerical': 0.002130099950843847, 'mean_word_length': 3.1548672566371683, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '346096', 'n_tokens_mistral': 3717, 'n_tokens_neox': 3569, 'n_words': 1236}