<commit_before># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Finds perf trybots that can run telemetry tests."""

import json
import logging
import os
import re
import subprocess
import sys
import urllib2

from telemetry.core import platform
from telemetry.core.platform import trybot_device
from telemetry.core import possible_browser
from telemetry import decorators

CHROMIUM_CONFIG_FILENAME = 'tools/run-perf-test.cfg'
BLINK_CONFIG_FILENAME = 'Tools/run-perf-test.cfg'
SUCCESS, NO_CHANGES, ERROR = range(3)
# Unsupported Perf bisect bots.
EXCLUDED_BOTS = {
    'win_xp_perf_bisect',
    'linux_perf_tester',
    'linux_perf_bisector',
    'win_perf_bisect_builder',
    'win_x64_perf_bisect_builder',
    'linux_perf_bisect_builder',
    'mac_perf_bisect_builder',
    'android_perf_bisect_builder'
}

INCLUDE_BOTS = [
    'trybot-all',
    'trybot-all-win',
    'trybot-all-mac',
    'trybot-all-linux',
    'trybot-all-android'
]

class TrybotError(Exception):

  def __str__(self):
    return '%s\nError running tryjob.' % self.args[0]


class PossibleTrybotBrowser(possible_browser.PossibleBrowser):
  """A script that sends a job to a trybot."""

  def __init__(self, browser_type, _):
    target_os = browser_type.split('-')[1]
    self._builder_names = _GetBuilderNames(browser_type)
    super(PossibleTrybotBrowser, self).__init__(browser_type, target_os, True)

  def Create(self, finder_options):
    raise NotImplementedError()

  def SupportsOptions(self, finder_options):
    if ((finder_options.device and finder_options.device != 'trybot') or
        finder_options.chrome_root or
        finder_options.cros_remote or
        finder_options.extensions_to_load or
        finder_options.profile_dir):
      return False
    return True

  def IsRemote(self):
    return True

  def _RunProcess(self, cmd):
    logging.debug('Running process: "%s"', ' '.join(cmd))
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = proc.communicate()
    returncode = proc.poll()
    return (returncode, out, err)

  def _UpdateConfigAndRunTryjob(self, bot_platform, cfg_file_path):
    """Updates perf config file, uploads changes and excutes perf try job.

    Args:
      bot_platform: Name of the platform to be generated.
      cfg_file_path: Perf config file path.

    Returns:
      (result, msg) where result is one of:
          SUCCESS if a tryjob was sent
          NO_CHANGES if there was nothing to try,
          ERROR if a tryjob was attempted but an error encountered
          and msg is an error message if an error was encountered, or rietveld
          url if success, otherwise throws TrybotError exception.
    """
    config = self._GetPerfConfig(bot_platform)
    try:
      config_file = open(cfg_file_path, 'w')
    except IOError:
      msg = 'Cannot find %s. Please run from src dir.' % cfg_file_path
      return (ERROR, msg)
    config_file.write('config = %s' % json.dumps(
        config, sort_keys=True, indent=2, separators=(',', ': ')))
    config_file.close()
    # Commit the config changes locally.
    returncode, out, err = self._RunProcess(
        ['git', 'commit', '-a', '-m', 'bisect config: %s' % bot_platform])
    if returncode:
      raise TrybotError('Could not commit bisect config change for %s,'
                        ' error %s' % (bot_platform, err))
    # Upload the CL to rietveld and run a try job.
    returncode, out, err = self._RunProcess([
        'git', 'cl', 'upload', '-f', '--bypass-hooks', '-m',
        'CL for perf tryjob on %s' % bot_platform
    ])
    if returncode:
      raise TrybotError('Could upload to rietveld for %s, error %s' %
                        (bot_platform, err))

    match = re.search(r'https://codereview.chromium.org/[\d]+', out)
    if not match:
      raise TrybotError('Could not upload CL to rietveld for %s! Output %s' %
                        (bot_platform, out))
    rietveld_url = match.group(0)
    # Generate git try command for available bots.
    git_try_command = ['git', 'cl', 'try', '-m', 'tryserver.chromium.perf']
    for bot in self._builder_names[bot_platform]:
      git_try_command.extend(['-b', bot])
    returncode, out, err = self._RunProcess(git_try_command)
    if returncode:
      raise TrybotError('Could not try CL for %s, error %s' %
                        (bot_platform, err))

    return (SUCCESS, rietveld_url)

  def _GetPerfConfig(self, bot_platform):
    """Generates the perf config for try job.

    Args:
      bot_platform: Name of the platform to be generated.

    Returns:
      A dictionary with perf config parameters.
    """
    # Generate the command line for the perf trybots
    target_arch = 'ia32'
    arguments = sys.argv
    if bot_platform in ['win', 'win-x64']:
      arguments[0] = 'python tools\\perf\\run_benchmark'
    else:
      arguments[0] = './tools/perf/run_benchmark'
    for index, arg in enumerate(arguments):
      if arg.startswith('--browser='):
        if bot_platform == 'android':
          arguments[index] = '--browser=android-chrome-shell'
        elif any('x64' in bot for bot in self._builder_names[bot_platform]):
          arguments[index] = '--browser=release_x64'
          target_arch = 'x64'
        else:
          arguments[index] = '--browser=release'
    command = ' '.join(arguments)

    return {
        'command': command,
        'repeat_count': '1',
        'max_time_minutes': '120',
        'truncate_percent': '0',
        'target_arch': target_arch,
    }

  def _AttemptTryjob(self, cfg_file_path):
    """Attempts to run a tryjob from the current directory.

    This is run once for chromium, and if it returns NO_CHANGES, once for blink.

    Args:
      cfg_file_path: Path to the config file for the try job.

    Returns:
      Returns SUCCESS if a tryjob was sent, NO_CHANGES if there was nothing to
      try, ERROR if a tryjob was attempted but an error encountered.
    """
    source_repo = 'chromium'
    if cfg_file_path == BLINK_CONFIG_FILENAME:
      source_repo = 'blink'<commit_msg>Make correction to Telemetry command for perf trybot.<commit_after># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Finds perf trybots that can run telemetry tests."""

import json
import logging
import os
import re
import subprocess
import sys
import urllib2

from telemetry.core import platform
from telemetry.core.platform import trybot_device
from telemetry.core import possible_browser
from telemetry import decorators

CHROMIUM_CONFIG_FILENAME = 'tools/run-perf-test.cfg'
BLINK_CONFIG_FILENAME = 'Tools/run-perf-test.cfg'
SUCCESS, NO_CHANGES, ERROR = range(3)
# Unsupported Perf bisect bots.
EXCLUDED_BOTS = {
    'win_xp_perf_bisect',
    'linux_perf_tester',
    'linux_perf_bisector',
    'win_perf_bisect_builder',
    'win_x64_perf_bisect_builder',
    'linux_perf_bisect_builder',
    'mac_perf_bisect_builder',
    'android_perf_bisect_builder'
}

INCLUDE_BOTS = [
    'trybot-all',
    'trybot-all-win',
    'trybot-all-mac',
    'trybot-all-linux',
    'trybot-all-android'
]

class TrybotError(Exception):

  def __str__(self):
    return '%s\nError running tryjob.' % self.args[0]


class PossibleTrybotBrowser(possible_browser.PossibleBrowser):
  """A script that sends a job to a trybot."""

  def __init__(self, browser_type, _):
    target_os = browser_type.split('-')[1]
    self._builder_names = _GetBuilderNames(browser_type)
    super(PossibleTrybotBrowser, self).__init__(browser_type, target_os, True)

  def Create(self, finder_options):
    raise NotImplementedError()

  def SupportsOptions(self, finder_options):
    if ((finder_options.device and finder_options.device != 'trybot') or
        finder_options.chrome_root or
        finder_options.cros_remote or
        finder_options.extensions_to_load or
        finder_options.profile_dir):
      return False
    return True

  def IsRemote(self):
    return True

  def _RunProcess(self, cmd):
    logging.debug('Running process: "%s"', ' '.join(cmd))
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = proc.communicate()
    returncode = proc.poll()
    return (returncode, out, err)

  def _UpdateConfigAndRunTryjob(self, bot_platform, cfg_file_path):
    """Updates perf config file, uploads changes and excutes perf try job.

    Args:
      bot_platform: Name of the platform to be generated.
      cfg_file_path: Perf config file path.

    Returns:
      (result, msg) where result is one of:
          SUCCESS if a tryjob was sent
          NO_CHANGES if there was nothing to try,
          ERROR if a tryjob was attempted but an error encountered
          and msg is an error message if an error was encountered, or rietveld
          url if success, otherwise throws TrybotError exception.
    """
    config = self._GetPerfConfig(bot_platform)
    try:
      config_file = open(cfg_file_path, 'w')
    except IOError:
      msg = 'Cannot find %s. Please run from src dir.' % cfg_file_path
      return (ERROR, msg)
    config_file.write('config = %s' % json.dumps(
        config, sort_keys=True, indent=2, separators=(',', ': ')))
    config_file.close()
    # Commit the config changes locally.
    returncode, out, err = self._RunProcess(
        ['git', 'commit', '-a', '-m', 'bisect config: %s' % bot_platform])
    if returncode:
      raise TrybotError('Could not commit bisect config change for %s,'
                        ' error %s' % (bot_platform, err))
    # Upload the CL to rietveld and run a try job.
    returncode, out, err = self._RunProcess([
        'git', 'cl', 'upload', '-f', '--bypass-hooks', '-m',
        'CL for perf tryjob on %s' % bot_platform
    ])
    if returncode:
      raise TrybotError('Could upload to rietveld for %s, error %s' %
                        (bot_platform, err))

    match = re.search(r'https://codereview.chromium.org/[\d]+', out)
    if not match:
      raise TrybotError('Could not upload CL to rietveld for %s! Output %s' %
                        (bot_platform, out))
    rietveld_url = match.group(0)
    # Generate git try command for available bots.
    git_try_command = ['git', 'cl', 'try', '-m', 'tryserver.chromium.perf']
    for bot in self._builder_names[bot_platform]:
      git_try_command.extend(['-b', bot])
    returncode, out, err = self._RunProcess(git_try_command)
    if returncode:
      raise TrybotError('Could not try CL for %s, error %s' %
                        (bot_platform, err))

    return (SUCCESS, rietveld_url)

  def _GetPerfConfig(self, bot_platform):
    """Generates the perf config for try job.

    Args:
      bot_platform: Name of the platform to be generated.

    Returns:
      A dictionary with perf config parameters.
    """
    # Generate the command line for the perf trybots
    target_arch = 'ia32'
    arguments = sys.argv
    if bot_platform in ['win', 'win-x64']:
      arguments[0] = 'python tools\\perf\\run_benchmark'
    else:
      arguments[0] = './tools/perf/run_benchmark'
    for index, arg in enumerate(arguments):
      if arg.startswith('--browser'):
        if bot_platform == 'android':
          arguments[index] = '--browser=android-chrome-shell'
        elif any('x64' in bot for bot in self._builder_names[bot_platform]):
          arguments[index] = '--browser=release_x64'
          target_arch = 'x64'
        else:
          arguments[index] = '--browser=release'
      if arg in _GetTrybotList():
        arguments[index] = ''

    command = ' '.join(arguments)

    return {
        'command': command,
        'repeat_count': '1',
        'max_time_minutes': '120',
        'truncate_percent': '0',
        'target_arch': target_arch,
    }

  def _AttemptTryjob(self, cfg_file_path):
    """Attempts to run a tryjob from the current directory.

    This is run once for chromium, and if it returns NO_CHANGES, once for blink.

    Args:
      cfg_file_path: Path to the config file for the try job.

    Returns:
      Returns SUCCESS if a tryjob was sent, NO_CHANGES if there was nothing to
      try, ERROR if a tryjob was attempted but an error encountered.
    """
    source_repo = 'chromium'
    if cfg_file_path == BLINK_CONFIG_FILENAME:
      source_repo = 'blink'
{'fraction_non_alphanumeric': 0.10435761376427079, 'fraction_numerical': 0.0046631291204373695, 'mean_word_length': 3.0399480350763235, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6816201', 'n_tokens_mistral': 3965, 'n_tokens_neox': 3835, 'n_words': 1157}