<commit_before>#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""
run talos tests in a virtualenv
"""

import os
import pprint
import copy
import re

from mozharness.base.config import parse_config_file
from mozharness.base.errors import PythonErrorList
from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL, FATAL
from mozharness.base.log import INFO, WARNING
from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options, INSTALLER_SUFFIXES
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.testing.errors import TinderBoxPrintRe
from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WORST_LEVEL_TUPLE
from mozharness.mozilla.buildbot import TBPL_RETRY, TBPL_FAILURE, TBPL_WARNING

TalosErrorList = PythonErrorList + [
    {'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
    {'substr': r'''FAIL: Graph server unreachable''', 'level': CRITICAL},
    {'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
    {'substr': r'''FAIL: failed to cleanup''', 'level': ERROR},
    {'substr': r'''erfConfigurator.py: Unknown error''', 'level': CRITICAL},
    {'substr': r'''talosError''', 'level': CRITICAL},
    {'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL},
    {'substr': r"""No such file or directory: 'browser_output.txt'""",
     'level': CRITICAL,
     'explanation': r"""Most likely the browser failed to launch, or the test was otherwise unsuccessful in even starting."""},
]

# TODO: check for running processes on script invocation


class TalosOutputParser(OutputParser):
    minidump_regex = re.compile(r'''talosError: "error executing: '(\S+) (\S+) (\S+)'"''')
    minidump_output = None
    worst_tbpl_status = TBPL_SUCCESS

    def parse_single_line(self, line):
        """ In Talos land, every line that starts with RETURN: needs to be
        printed with a TinderboxPrint:"""
        if line.startswith("RETURN:"):
            line.replace("RETURN:", "TinderboxPrint:")
        m = self.minidump_regex.search(line)
        if m:
            self.minidump_output = (m.group(1), m.group(2), m.group(3))

        # now let's check if buildbot should retry
        harness_retry_re = TinderBoxPrintRe['harness_error']['retry_regex']
        if harness_retry_re.search(line):
            self.critical(' %s' % line)
            self.worst_log_level = self.worst_level(CRITICAL,
                                                    self.worst_log_level)
            self.worst_tbpl_status = self.worst_level(
                TBPL_RETRY, self.worst_tbpl_status,
                levels=TBPL_WORST_LEVEL_TUPLE
            )
            return  # skip base parse_single_line
        super(TalosOutputParser, self).parse_single_line(line)


talos_config_options = [
    [["-a", "--tests"],
     {'action': 'extend',
      "dest": "tests",
      "default": [],
      "help": "Specify the tests to run"
      }],
    [["--results-url"],
     {'action': 'store',
      'dest': 'results_url',
      'default': None,
      'help': "URL to send results to"
      }],
]


class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
    """
    install and run Talos tests:
    https://wiki.mozilla.org/Buildbot/Talos
    """

    config_options = [
        [["--talos-url"],
         {"action": "store",
          "dest": "talos_url",
          "default": "https://hg.mozilla.org/build/talos/archive/tip.tar.gz",
          "help": "Specify the talos package url"
          }],
        [["--use-talos-json"],
         {"action": "store_true",
          "dest": "use_talos_json",
          "default": False,
          "help": "Use talos config from talos.json"
          }],
        [["--suite"],
         {"action": "store",
          "dest": "suite",
          "help": "Talos suite to run (from talos json)"
          }],
        [["--branch-name"],
         {"action": "store",
          "dest": "branch",
          "help": "Graphserver branch to report to"
          }],
        [["--system-bits"],
         {"action": "store",
          "dest": "system_bits",
          "type": "choice",
          "default": "32",
          "choices": ['32', '64'],
          "help": "Testing 32 or 64 (for talos json plugins)"
          }],
        [["--add-option"],
         {"action": "extend",
          "dest": "talos_extra_options",
          "default": None,
          "help": "extra options to talos"
          }],
    ] + talos_config_options + testing_config_options + \
        copy.deepcopy(blobupload_config_options)

    def __init__(self, **kwargs):
        kwargs.setdefault('config_options', self.config_options)
        kwargs.setdefault('all_actions', ['clobber',
                                          'read-buildbot-config',
                                          'download-and-extract',
                                          'clone-talos',
                                          'create-virtualenv',
                                          'install',
                                          'run-tests',
                                          ])
        kwargs.setdefault('default_actions', ['clobber',
                                              'download-and-extract',
                                              'clone-talos',
                                              'create-virtualenv',
                                              'install',
                                              'run-tests',
                                              ])
        kwargs.setdefault('config', {})
        kwargs['config'].setdefault('virtualenv_modules', ["talos", "mozinstall"])
        super(Talos, self).__init__(**kwargs)

        self.workdir = self.query_abs_dirs()['abs_work_dir']  # convenience

        # results output
        self.results_url = self.config.get('results_url')
        if self.results_url is None:
            # use a results_url by default based on the class name in the working directory
            self.results_url = 'file://%s' % os.path.join(self.workdir, self.__class__.__name__.lower() + '.txt')
        self.installer_url = self.config.get("installer_url")
        self.talos_json_url = self.config.get("talos_json_url")
        self.talos_json = self.config.get("talos_json")
        self.talos_json_config = self.config.get("talos_json_config")
        self.talos_path = os.path.join(self.workdir, 'talos_repo')
        self.has_cloned_talos = False
        self.tests = None
        self.pagesets_url = None
        self.pagesets_parent_dir_path = None
        self.pagesets_manifest_path = None
        self.abs_pagesets_paths = None
        self.pagesets_manifest_filename = None
        self.pagesets_manifest_parent_path = None
        if 'run-tests' in self.actions:
            self.preflight_run_tests()

    def query_abs_dirs(self):
        c = self.config
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = super(Talos, self).query_abs_dirs()
        abs_dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
        if c.get('metro_harness_path_frmt'):
            abs_dirs['abs_metro_path'] = c['metro_harness_path_frmt'] % {
                "metro_base_path": os.path.join(abs_dirs['abs_work_dir'],
                                                'talos_repo')
            }
        self.abs_dirs = abs_dirs
        return self.abs_dirs

    def query_talos_json_url(self):
        """Hacky, but I haven't figured out a better way to get the
        talos json url before we install the build.

        We can't get this information after we install the build, because
        we have to create the virtualenv to use mozinstall, and talos_url
        is specified in the talos json.
        """
        if self.talos_json_url:
            return self.talos_json_url
        self.info("Guessing talos json url...")
        if not self.installer_url:
            self.read_buildbot_config()
            self.postflight_read_buildbot_config()
            if not self.installer_url:
                self.fatal("Can't figure out talos_json_url without an installer_url!")
        for suffix in INSTALLER_SUFFIXES:
            if self.installer_url.endswith(suffix):
                build_txt_url = self.installer_url[:-len(suffix)] + '.txt'
                break
        else:
            self.fatal("Can't figure out talos_json_url from installer_url %s!" % self.installer_url)
        build_txt_file = self.download_file(build_txt_url, parent_dir=self.workdir)
        if not build_txt_file:
            self.fatal("Can't download %s to guess talos_json_url!" % build_txt_url)
        # HG hardcode?
        revision_re = re.compile(r'''([a-zA-Z]+://.+)/rev/([0-9a-fA-F]{10})''')
        contents = self.read_from_file(build_txt_file, error_level=FATAL).splitlines()
        for line in contents:
            m = revision_re.match(line)
            if m:
                break
        else:
            self.fatal("Can't figure out talos_json_url from %s!" % build_txt_file)
        self.talos_json_url = "%s/raw-file/%s/testing/talos/talos.json" % (m.group(1), m.group(2))
        return self.talos_json_url

    def download_talos_json(self):
        talos_json_url = self.query_talos_json_url()
        self.talos_json = self.download_file(talos_json_url,
                                             parent_dir=self.workdir,
                                             error_level=FATAL)

    def query_talos_json_config(self):
        """Return the talos json config; download and read from the
        talos_json_url if need be."""
        if self.talos_json_config:
            return self.talos_json_config
        c = self.config
        if not c['use_talos_json']:
            return
        if not c['suite']:
            self.fatal("To use talos_json, you must define use_talos_json, suite.")
            return
        if not self.talos_json:
            talos_json_url = self.query_talos_json_url()
            if not talos_json_url:
                self.fatal("Can't download talos_json without a talos_json_url!")
            self.download_talos_json()
        self.talos_json_config = parse_config_file(self.talos_json)
        self.info(pprint.pformat(self.talos_json_config))
        return self.talos_json_config

    def query_tests(self):
        """Determine if we have tests to run.

        Currently talos json will take precedence over config and command
        line options; if that's not a good default we can switch the order.
        """
        if self.tests is not None:
            return self.tests
        c = self.config
        if c['use_talos_json']:
            if not c['suite']:
                self.fatal("Can't use_talos_json without a --suite!")
            talos_config = self.query_talos_json_config()
            try:
                self.tests = talos_config['suites'][c['suite']]['tests']
            except KeyError, e:
                self.error("Badly formed talos_json for suite %s; KeyError trying to access talos_config['suites'][%s]['tests']: %s" % (c['suite'], c['suite'], str(e)))
        elif c['tests']:
            self.tests = c['tests']
        # Ignore these tests, specifically so we can not run a11yr on osx
        if c.get('ignore_tests'):
            for test in c['ignore_tests']:
                if test in self.tests:
                    del self.tests[self.tests.index(test)]
        return self.tests

    def query_talos_options(self):
        options = []
        c = self.config
        if self.query_talos_json_config():
            options += self.talos_json_config['suites'][c['suite']].get('talos_options', [])
        if c.get('talos_extra_options'):
            options += c['talos_extra_options']
        return options

    def query_talos_repo(self):
        """Where do we install the talos python package from?
        This needs to be overrideable by the talos json.
        """
        default_repo = "https://hg.mozilla.org/build/talos"
        if self.query_talos_json_config():
            return self.talos_json_config.get('global', {}).get('talos_repo', default_repo)
        else:
            return self.config.get('talos_repo', default_repo)

    def query_talos_revision(self):
        """Which talos revision do we want to use?
        This needs to be overrideable by the talos json.
        """
        if self.query_talos_json_config():
            return self.talos_json_config['global']['talos_revision']
        else:
            return self.config.get('talos_revision')

    def query_pagesets_url(self):
        """Certain suites require external pagesets to be downloaded and
        extracted.
        """
        if self.pagesets_url:
            return self.pagesets_url
        if self.query_talos_json_config():
            self.pagesets_url = self.talos_json_config['suites'][self.config['suite']].get('pagesets_url')
            return self.pagesets_url

    def query_pagesets_parent_dir_path(self):
        """ We have to copy the pageset into the webroot separately.

        Helper method to avoid hardcodes.
        """
        if self.pagesets_parent_dir_path:
            return self.pagesets_parent_dir_path
        if self.query_talos_json_config():
            self.pagesets_parent_dir_path = self.talos_json_config['suites'][self.config['suite']].get('pagesets_parent_dir_path')
            return self.pagesets_parent_dir_path

    def query_pagesets_manifest_path(self):
        """ We have to copy the tp manifest from webroot to talos root when
        those two directories aren't the same, until bug 795172 is fixed.

        Helper method to avoid hardcodes.
        """
        if self.pagesets_manifest_path:
            return self.pagesets_manifest_path
        if self.query_talos_json_config():
            self.pagesets_manifest_path = self.talos_json_config['suites'][self.config['suite']].get('pagesets_manifest_path')
            return self.pagesets_manifest_path

    def query_pagesets_manifest_filename(self):
        if self.pagesets_manifest_filename:
            return self.pagesets_manifest_filename
        else:
            manifest_path = self.query_pagesets_manifest_path()
            self.pagesets_manifest_filename = os.path.basename(manifest_path)
            return self.pagesets_manifest_filename

    def query_pagesets_manifest_parent_path(self):
        if self.pagesets_manifest_parent_path:
            return self.pagesets_manifest_parent_path
        if self.query_talos_json_config():
            manifest_path = self.query_pagesets_manifest_path()
            self.pagesets_manifest_parent_path = os.path.dirname(manifest_path)
            return self.pagesets_manifest_parent_path

    def query_abs_pagesets_paths(self):
        """ Returns a bunch of absolute pagesets directory paths.
        We need this to make the dir and copy the manifest to the local dir.
        """
        if self.abs_pagesets_paths:
            return self.abs_pagesets_paths
        else:
            paths = {}
            manifest_parent_path = self.query_pagesets_manifest_parent_path()
            paths['pagesets_manifest_parent'] = os.path.join(self.talos_path, manifest_parent_path)

            manifest_path = self.query_pagesets_manifest_path()
            paths['pagesets_manifest'] = os.path.join(self.talos_path, manifest_path)

            self.abs_pagesets_paths = paths
            return self.abs_pagesets_paths

    def talos_options(self, args=None, **kw):
        """return options to talos"""
        # binary path
        binary_path = self.binary_path or self.config.get('binary_path')
        if not binary_path:
            self.fatal("Talos requires a path to the binary.  You can specify binary_path or add download-and-extract to your action list.")

        # talos options
        options = ['-v', ]  # hardcoded options (for now)
        if self.config.get('python_webserver', True):
            options.append('--develop')
        # talos can't gather data if the process name ends with '.exe'
        if binary_path.endswith('.exe'):
            binary_path = binary_path[:-4]
        kw_options = {'output': 'talos.yml',  # options overwritten from **kw
                      'executablePath': binary_path,
                      'results_url': self.results_url}
        kw_options['activeTests'] = self.query_tests()
        if self.config.get('title'):
            kw_options['title'] = self.config['title']
        if self.config.get('branch'):
            kw_options['branchName'] = self.config['branch']
        if self.symbols_path:
            kw_options['symbolsPath'] = self.symbols_path
        kw_options.update(kw)
        # talos expects tests to be in the format (e.g.) 'ts:tp5:tsvg'
        tests = kw_options.get('activeTests')
        if tests and not isinstance(tests, basestring):
            tests = ':'.join(tests)  # Talos expects this format
            kw_options['activeTests'] = tests
        for key, value in kw_options.items():
            options.extend(['--%s' % key, value])
        # add datazilla results urls
        for url in self.config.get('datazilla_urls', []):
            options.extend(['--datazilla-url', url])
        # add datazilla authfile
        authfile = self.config.get('datazilla_authfile')
        if authfile:
            options.extend(['--authfile', authfile])
        # extra arguments
        if args is None:
            args = self.query_talos_options()
        options += args

        return options

    def talos_conf_path(self, conf):<commit_msg>Bug 967630 - Add an option to trychooser to select Talos profiling options, r=jlund<commit_after>#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""
run talos tests in a virtualenv
"""

import os
import pprint
import copy
import re

from mozharness.base.config import parse_config_file
from mozharness.base.errors import PythonErrorList
from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL, FATAL
from mozharness.base.log import INFO, WARNING
from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options, INSTALLER_SUFFIXES
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.testing.errors import TinderBoxPrintRe
from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WORST_LEVEL_TUPLE
from mozharness.mozilla.buildbot import TBPL_RETRY, TBPL_FAILURE, TBPL_WARNING

TalosErrorList = PythonErrorList + [
    {'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
    {'substr': r'''FAIL: Graph server unreachable''', 'level': CRITICAL},
    {'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
    {'substr': r'''FAIL: failed to cleanup''', 'level': ERROR},
    {'substr': r'''erfConfigurator.py: Unknown error''', 'level': CRITICAL},
    {'substr': r'''talosError''', 'level': CRITICAL},
    {'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL},
    {'substr': r"""No such file or directory: 'browser_output.txt'""",
     'level': CRITICAL,
     'explanation': r"""Most likely the browser failed to launch, or the test was otherwise unsuccessful in even starting."""},
]

# TODO: check for running processes on script invocation


class TalosOutputParser(OutputParser):
    minidump_regex = re.compile(r'''talosError: "error executing: '(\S+) (\S+) (\S+)'"''')
    minidump_output = None
    worst_tbpl_status = TBPL_SUCCESS

    def parse_single_line(self, line):
        """ In Talos land, every line that starts with RETURN: needs to be
        printed with a TinderboxPrint:"""
        if line.startswith("RETURN:"):
            line.replace("RETURN:", "TinderboxPrint:")
        m = self.minidump_regex.search(line)
        if m:
            self.minidump_output = (m.group(1), m.group(2), m.group(3))

        # now let's check if buildbot should retry
        harness_retry_re = TinderBoxPrintRe['harness_error']['retry_regex']
        if harness_retry_re.search(line):
            self.critical(' %s' % line)
            self.worst_log_level = self.worst_level(CRITICAL,
                                                    self.worst_log_level)
            self.worst_tbpl_status = self.worst_level(
                TBPL_RETRY, self.worst_tbpl_status,
                levels=TBPL_WORST_LEVEL_TUPLE
            )
            return  # skip base parse_single_line
        super(TalosOutputParser, self).parse_single_line(line)


talos_config_options = [
    [["-a", "--tests"],
     {'action': 'extend',
      "dest": "tests",
      "default": [],
      "help": "Specify the tests to run"
      }],
    [["--results-url"],
     {'action': 'store',
      'dest': 'results_url',
      'default': None,
      'help': "URL to send results to"
      }],
]


class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
    """
    install and run Talos tests:
    https://wiki.mozilla.org/Buildbot/Talos
    """

    config_options = [
        [["--talos-url"],
         {"action": "store",
          "dest": "talos_url",
          "default": "https://hg.mozilla.org/build/talos/archive/tip.tar.gz",
          "help": "Specify the talos package url"
          }],
        [["--use-talos-json"],
         {"action": "store_true",
          "dest": "use_talos_json",
          "default": False,
          "help": "Use talos config from talos.json"
          }],
        [["--suite"],
         {"action": "store",
          "dest": "suite",
          "help": "Talos suite to run (from talos json)"
          }],
        [["--branch-name"],
         {"action": "store",
          "dest": "branch",
          "help": "Graphserver branch to report to"
          }],
        [["--system-bits"],
         {"action": "store",
          "dest": "system_bits",
          "type": "choice",
          "default": "32",
          "choices": ['32', '64'],
          "help": "Testing 32 or 64 (for talos json plugins)"
          }],
        [["--add-option"],
         {"action": "extend",
          "dest": "talos_extra_options",
          "default": None,
          "help": "extra options to talos"
          }],
        [["--spsProfile"], {
            "dest": "sps_profile",
            "action": "store_true",
            "default": False,
            "help": "Whether or not to profile the test run and save the profile results"
        }],
        [["--spsProfileInterval"], {
            "dest": "sps_profile_interval",
            "type": "int",
            "default": 0,
            "help": "The interval between samples taken by the profiler (milliseconds)"
        }],
    ] + talos_config_options + testing_config_options + \
        copy.deepcopy(blobupload_config_options)

    def __init__(self, **kwargs):
        kwargs.setdefault('config_options', self.config_options)
        kwargs.setdefault('all_actions', ['clobber',
                                          'read-buildbot-config',
                                          'download-and-extract',
                                          'clone-talos',
                                          'create-virtualenv',
                                          'install',
                                          'run-tests',
                                          ])
        kwargs.setdefault('default_actions', ['clobber',
                                              'download-and-extract',
                                              'clone-talos',
                                              'create-virtualenv',
                                              'install',
                                              'run-tests',
                                              ])
        kwargs.setdefault('config', {})
        kwargs['config'].setdefault('virtualenv_modules', ["talos", "mozinstall"])
        super(Talos, self).__init__(**kwargs)

        self.workdir = self.query_abs_dirs()['abs_work_dir']  # convenience

        # results output
        self.results_url = self.config.get('results_url')
        if self.results_url is None:
            # use a results_url by default based on the class name in the working directory
            self.results_url = 'file://%s' % os.path.join(self.workdir, self.__class__.__name__.lower() + '.txt')
        self.installer_url = self.config.get("installer_url")
        self.talos_json_url = self.config.get("talos_json_url")
        self.talos_json = self.config.get("talos_json")
        self.talos_json_config = self.config.get("talos_json_config")
        self.talos_path = os.path.join(self.workdir, 'talos_repo')
        self.has_cloned_talos = False
        self.tests = None
        self.pagesets_url = None
        self.pagesets_parent_dir_path = None
        self.pagesets_manifest_path = None
        self.abs_pagesets_paths = None
        self.pagesets_manifest_filename = None
        self.pagesets_manifest_parent_path = None
        self.sps_profile = self.config.get('sps_profile')
        self.sps_profile_interval = self.config.get('sps_profile_interval')
        if 'run-tests' in self.actions:
            self.preflight_run_tests()

    # We accept some configuration options from the try commit message in the format mozharness: <options>
    # Example try commit message:
    #   mozharness: --spsProfile try: <stuff>
    def query_sps_profile_options(self):
        sps_results = []
        if self.buildbot_config:
            # this is inside automation
            # now let's see if we added spsProfile specs in the commit message
            junk, junk, opts = self.buildbot_config['sourcestamp']['changes'][-1]['comments'].partition('mozharness:')
            if opts:
              opts = re.sub(r'\w+:.*', '', opts).strip().split(' ')
              if "--spsProfile" in opts:
                  # overwrite whatever was set here.
                  self.sps_profile = True
              try:
                    idx = opts.index('--spsProfileInterval')
                    if len(opts) > idx + 1:
                        self.sps_profile_interval = opts[idx + 1]
              except ValueError:
                  pass
        # finally, if sps_profile is set, we add that to the talos options
        if self.sps_profile:
            sps_results.append('--spsProfile')
            if self.sps_profile_interval:
                sps_results.extend(
                    ['--spsProfileInterval', str(self.sps_profile_interval)]
                )
        return sps_results

    def query_abs_dirs(self):
        c = self.config
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = super(Talos, self).query_abs_dirs()
        abs_dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
        if c.get('metro_harness_path_frmt'):
            abs_dirs['abs_metro_path'] = c['metro_harness_path_frmt'] % {
                "metro_base_path": os.path.join(abs_dirs['abs_work_dir'],
                                                'talos_repo')
            }
        self.abs_dirs = abs_dirs
        return self.abs_dirs

    def query_talos_json_url(self):
        """Hacky, but I haven't figured out a better way to get the
        talos json url before we install the build.

        We can't get this information after we install the build, because
        we have to create the virtualenv to use mozinstall, and talos_url
        is specified in the talos json.
        """
        if self.talos_json_url:
            return self.talos_json_url
        self.info("Guessing talos json url...")
        if not self.installer_url:
            self.read_buildbot_config()
            self.postflight_read_buildbot_config()
            if not self.installer_url:
                self.fatal("Can't figure out talos_json_url without an installer_url!")
        for suffix in INSTALLER_SUFFIXES:
            if self.installer_url.endswith(suffix):
                build_txt_url = self.installer_url[:-len(suffix)] + '.txt'
                break
        else:
            self.fatal("Can't figure out talos_json_url from installer_url %s!" % self.installer_url)
        build_txt_file = self.download_file(build_txt_url, parent_dir=self.workdir)
        if not build_txt_file:
            self.fatal("Can't download %s to guess talos_json_url!" % build_txt_url)
        # HG hardcode?
        revision_re = re.compile(r'''([a-zA-Z]+://.+)/rev/([0-9a-fA-F]{10})''')
        contents = self.read_from_file(build_txt_file, error_level=FATAL).splitlines()
        for line in contents:
            m = revision_re.match(line)
            if m:
                break
        else:
            self.fatal("Can't figure out talos_json_url from %s!" % build_txt_file)
        self.talos_json_url = "%s/raw-file/%s/testing/talos/talos.json" % (m.group(1), m.group(2))
        return self.talos_json_url

    def download_talos_json(self):
        talos_json_url = self.query_talos_json_url()
        self.talos_json = self.download_file(talos_json_url,
                                             parent_dir=self.workdir,
                                             error_level=FATAL)

    def query_talos_json_config(self):
        """Return the talos json config; download and read from the
        talos_json_url if need be."""
        if self.talos_json_config:
            return self.talos_json_config
        c = self.config
        if not c['use_talos_json']:
            return
        if not c['suite']:
            self.fatal("To use talos_json, you must define use_talos_json, suite.")
            return
        if not self.talos_json:
            talos_json_url = self.query_talos_json_url()
            if not talos_json_url:
                self.fatal("Can't download talos_json without a talos_json_url!")
            self.download_talos_json()
        self.talos_json_config = parse_config_file(self.talos_json)
        self.info(pprint.pformat(self.talos_json_config))
        return self.talos_json_config

    def query_tests(self):
        """Determine if we have tests to run.

        Currently talos json will take precedence over config and command
        line options; if that's not a good default we can switch the order.
        """
        if self.tests is not None:
            return self.tests
        c = self.config
        if c['use_talos_json']:
            if not c['suite']:
                self.fatal("Can't use_talos_json without a --suite!")
            talos_config = self.query_talos_json_config()
            try:
                self.tests = talos_config['suites'][c['suite']]['tests']
            except KeyError, e:
                self.error("Badly formed talos_json for suite %s; KeyError trying to access talos_config['suites'][%s]['tests']: %s" % (c['suite'], c['suite'], str(e)))
        elif c['tests']:
            self.tests = c['tests']
        # Ignore these tests, specifically so we can not run a11yr on osx
        if c.get('ignore_tests'):
            for test in c['ignore_tests']:
                if test in self.tests:
                    del self.tests[self.tests.index(test)]
        return self.tests

    def query_talos_options(self):
        options = []
        c = self.config
        if self.query_talos_json_config():
            options += self.talos_json_config['suites'][c['suite']].get('talos_options', [])
        if c.get('talos_extra_options'):
            options += c['talos_extra_options']
        return options

    def query_talos_repo(self):
        """Where do we install the talos python package from?
        This needs to be overrideable by the talos json.
        """
        default_repo = "https://hg.mozilla.org/build/talos"
        if self.query_talos_json_config():
            return self.talos_json_config.get('global', {}).get('talos_repo', default_repo)
        else:
            return self.config.get('talos_repo', default_repo)

    def query_talos_revision(self):
        """Which talos revision do we want to use?
        This needs to be overrideable by the talos json.
        """
        if self.query_talos_json_config():
            return self.talos_json_config['global']['talos_revision']
        else:
            return self.config.get('talos_revision')

    def query_pagesets_url(self):
        """Certain suites require external pagesets to be downloaded and
        extracted.
        """
        if self.pagesets_url:
            return self.pagesets_url
        if self.query_talos_json_config():
            self.pagesets_url = self.talos_json_config['suites'][self.config['suite']].get('pagesets_url')
            return self.pagesets_url

    def query_pagesets_parent_dir_path(self):
        """ We have to copy the pageset into the webroot separately.

        Helper method to avoid hardcodes.
        """
        if self.pagesets_parent_dir_path:
            return self.pagesets_parent_dir_path
        if self.query_talos_json_config():
            self.pagesets_parent_dir_path = self.talos_json_config['suites'][self.config['suite']].get('pagesets_parent_dir_path')
            return self.pagesets_parent_dir_path

    def query_pagesets_manifest_path(self):
        """ We have to copy the tp manifest from webroot to talos root when
        those two directories aren't the same, until bug 795172 is fixed.

        Helper method to avoid hardcodes.
        """
        if self.pagesets_manifest_path:
            return self.pagesets_manifest_path
        if self.query_talos_json_config():
            self.pagesets_manifest_path = self.talos_json_config['suites'][self.config['suite']].get('pagesets_manifest_path')
            return self.pagesets_manifest_path

    def query_pagesets_manifest_filename(self):
        if self.pagesets_manifest_filename:
            return self.pagesets_manifest_filename
        else:
            manifest_path = self.query_pagesets_manifest_path()
            self.pagesets_manifest_filename = os.path.basename(manifest_path)
            return self.pagesets_manifest_filename

    def query_pagesets_manifest_parent_path(self):
        if self.pagesets_manifest_parent_path:
            return self.pagesets_manifest_parent_path
        if self.query_talos_json_config():
            manifest_path = self.query_pagesets_manifest_path()
            self.pagesets_manifest_parent_path = os.path.dirname(manifest_path)
            return self.pagesets_manifest_parent_path

    def query_abs_pagesets_paths(self):
        """ Returns a bunch of absolute pagesets directory paths.
        We need this to make the dir and copy the manifest to the local dir.
        """
        if self.abs_pagesets_paths:
            return self.abs_pagesets_paths
        else:
            paths = {}
            manifest_parent_path = self.query_pagesets_manifest_parent_path()
            paths['pagesets_manifest_parent'] = os.path.join(self.talos_path, manifest_parent_path)

            manifest_path = self.query_pagesets_manifest_path()
            paths['pagesets_manifest'] = os.path.join(self.talos_path, manifest_path)

            self.abs_pagesets_paths = paths
            return self.abs_pagesets_paths

    def talos_options(self, args=None, **kw):
        """return options to talos"""
        # binary path
        binary_path = self.binary_path or self.config.get('binary_path')
        if not binary_path:
            self.fatal("Talos requires a path to the binary.  You can specify binary_path or add download-and-extract to your action list.")

        # talos options
        options = ['-v', ]  # hardcoded options (for now)
        if self.config.get('python_webserver', True):
            options.append('--develop')
        # talos can't gather data if the process name ends with '.exe'
        if binary_path.endswith('.exe'):
            binary_path = binary_path[:-4]
        kw_options = {'output': 'talos.yml',  # options overwritten from **kw
                      'executablePath': binary_path,
                      'results_url': self.results_url}
        kw_options['activeTests'] = self.query_tests()
        if self.config.get('title'):
            kw_options['title'] = self.config['title']
        if self.config.get('branch'):
            kw_options['branchName'] = self.config['branch']
        if self.symbols_path:
            kw_options['symbolsPath'] = self.symbols_path
        kw_options.update(kw)
        # talos expects tests to be in the format (e.g.) 'ts:tp5:tsvg'
        tests = kw_options.get('activeTests')
        if tests and not isinstance(tests, basestring):
            tests = ':'.join(tests)  # Talos expects this format
            kw_options['activeTests'] = tests
        for key, value in kw_options.items():
            options.extend(['--%s' % key, value])
        # add datazilla results urls
        for url in self.config.get('datazilla_urls', []):
            options.extend(['--datazilla-url', url])
        # add datazilla authfile
        authfile = self.config.get('datazilla_authfile')
        if authfile:
            options.extend(['--authfile', authfile])
        # configure profiling options
        options.extend(self.query_sps_profile_options())
        # extra arguments
        if args is None:
            args = self.query_talos_options()
        options += args

        return options

    def talos_conf_path(self, conf):
{'fraction_non_alphanumeric': 0.11032356661045531, 'fraction_numerical': 0.002002529510961214, 'mean_word_length': 2.1979271991911022, 'pattern_counts': {'":': 64, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 88, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4920425', 'n_tokens_mistral': 11245, 'n_tokens_neox': 10900, 'n_words': 2542}