<commit_before>#!/usr/bin/env python
# Mozilla licence shtuff

import sys
import os
import glob
import re
import tempfile
from datetime import datetime
import urllib2
import urlparse
import time
import xml.dom.minidom
import functools

try:
    import simplejson as json
    assert json
except ImportError:
    import json

# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))

# import the guts
from mozharness.base.config import parse_config_file
from mozharness.base.script import BaseScript
from mozharness.base.vcs.vcsbase import VCSMixin
from mozharness.base.transfer import TransferMixin
from mozharness.base.errors import MakefileErrorList
from mozharness.base.log import WARNING, ERROR, INFO, FATAL
from mozharness.mozilla.l10n.locales import GaiaLocalesMixin, LocalesMixin
from mozharness.mozilla.mock import MockMixin
from mozharness.mozilla.tooltool import TooltoolMixin
from mozharness.mozilla.buildbot import BuildbotMixin
from mozharness.mozilla.purge import PurgeMixin
from mozharness.mozilla.signing import SigningMixin
from mozharness.mozilla.repo_manifest import (load_manifest, rewrite_remotes,
                                              remove_project, get_project, get_remote)

# B2G builds complain about java...but it doesn't seem to be a problem
# Let's turn those into WARNINGS instead
B2GMakefileErrorList = MakefileErrorList + [
    {'substr': r'''NS_ERROR_FILE_ALREADY_EXISTS: Component returned failure code''', 'level': ERROR},
]
B2GMakefileErrorList.insert(0, {'substr': r'/bin/bash: java: command not found', 'level': WARNING})


def map_remote(r, mappings):
    """
    Helper function for mapping git remotes
    """
    remote = r.getAttribute('fetch')
    if remote in mappings:
        r.setAttribute('fetch', mappings[remote])
        return r
    return None


class B2GBuild(LocalesMixin, MockMixin, PurgeMixin, BaseScript, VCSMixin, TooltoolMixin,
               TransferMixin, BuildbotMixin, GaiaLocalesMixin, SigningMixin):
    config_options = [
        [["--repo"], {
            "dest": "repo",
            "help": "which gecko repo to check out",
        }],
        [["--gonk-snapshot"], {
            "dest": "gonk_snapshot_url",
            "help": "override the gonk snapshot specified in the build config",
        }],
        [["--target"], {
            "dest": "target",
            "help": "specify which build type to do",
        }],
        [["--b2g-config-dir"], {
            "dest": "b2g_config_dir",
            "help": "specify which in-tree config directory to use, relative to b2g/config/ (defaults to --target)",
        }],
        [["--gecko-config"], {
            "dest": "gecko_config",
            "help": "specfiy alternate location for gecko config",
        }],
        [["--disable-ccache"], {
            "dest": "ccache",
            "action": "store_false",
            "help": "disable ccache",
        }],
        [["--gaia-languages-file"], {
            "dest": "gaia_languages_file",
            "help": "languages file for gaia multilocale profile",
        }],
        [["--gecko-languages-file"], {
            "dest": "locales_file",
            "help": "languages file for gecko multilocale",
        }],
        [["--gecko-l10n-base-dir"], {
            "dest": "l10n_dir",
            "help": "dir to clone gecko l10n repos into, relative to the work directory",
        }],
        [["--merge-locales"], {
            "dest": "merge_locales",
            "help": "Dummy option to keep from burning. We now always merge",
        }],
        [["--variant"], {
            "dest": "variant",
            "help": "b2g build variant. overrides gecko config's value",
        }],
        [["--checkout-revision"], {
            "dest": "checkout_revision",
            "help": "checkout a specific gecko revision.",
        }],
        [["--additional-source-tarballs"], {
            "action": "extend",
            "type": "string",
            "dest": "additional_source_tarballs",
            "help": "Additional source tarballs to extract",
        }],
        [["--update-channel"], {
            "dest": "update_channel",
            "help": "b2g update channel",
        }],
    ]

    def __init__(self, require_config_file=False):
        self.gecko_config = None
        self.buildid = None
        self.wrote_b2g_config = False
        LocalesMixin.__init__(self)
        BaseScript.__init__(self,
                            config_options=self.config_options,
                            all_actions=[
                                'clobber',
                                'checkout-sources',
                                # Deprecated
                                'checkout-gecko',
                                'download-gonk',
                                'unpack-gonk',
                                'checkout-gaia',
                                'checkout-gaia-l10n',
                                'checkout-gecko-l10n',
                                'checkout-compare-locales',
                                # End deprecated
                                'get-blobs',
                                'update-source-manifest',
                                'build',
                                'build-symbols',
                                'make-updates',
                                'build-update-testdata',
                                'prep-upload',
                                'upload',
                                'make-update-xml',
                                'upload-updates',
                                'make-socorro-json',
                                'upload-source-manifest',
                            ],
                            default_actions=[
                                'checkout-sources',
                                'get-blobs',
                                'build',
                            ],
                            require_config_file=require_config_file,

                            # Default configuration
                            config={
                                'default_vcs': 'hgtool',
                                'vcs_share_base': os.environ.get('HG_SHARE_BASE_DIR'),
                                'ccache': True,
                                'buildbot_json_path': os.environ.get('PROPERTIES_FILE'),
                                'tooltool_servers': None,
                                'ssh_key': None,
                                'ssh_user': None,
                                'upload_remote_host': None,
                                'upload_remote_basepath': None,
                                'enable_try_uploads': False,
                                'tools_repo': 'http://hg.mozilla.org/build/tools',
                                'locales_dir': 'gecko/b2g/locales',
                                'l10n_dir': 'gecko-l10n',
                                'ignore_locales': ['en-US', 'multi'],
                                'locales_file': 'gecko/b2g/locales/all-locales',
                                'mozilla_dir': 'build/gecko',
                                'objdir': 'build/objdir-gecko',
                                'merge_locales': True,
                                'compare_locales_repo': 'http://hg.mozilla.org/build/compare-locales',
                                'compare_locales_rev': 'RELEASE_AUTOMATION',
                                'compare_locales_vcs': 'hgtool',
                                'repo_repo': "https://git.mozilla.org/external/google/gerrit/git-repo.git",
                                'repo_remote_mappings': {},
                                'update_channel': 'nightly',
                            },
                            )

        dirs = self.query_abs_dirs()
        self.objdir = os.path.join(dirs['work_dir'], 'objdir-gecko')
        self.marfile = "%s/dist/b2g-update/b2g-gecko-update.mar" % self.objdir
        self.application_ini = os.path.join(
            dirs['work_dir'], 'out', 'target', 'product',
            self.config['target'], 'system', 'b2g', 'application.ini')

    def _pre_config_lock(self, rw_config):
        super(B2GBuild, self)._pre_config_lock(rw_config)

        if self.buildbot_config is None:
            self.info("Reading buildbot build properties...")
            self.read_buildbot_config()

        if 'target' not in self.config:
            self.fatal("Must specify --target!")

        if not (self.buildbot_config and 'properties' in self.buildbot_config) and 'repo' not in self.config:
            self.fatal("Must specify --repo")

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = LocalesMixin.query_abs_dirs(self)

        c = self.config
        dirs = {
            'src': os.path.join(c['work_dir'], 'gecko'),
            'work_dir': abs_dirs['abs_work_dir'],
            'testdata_dir': os.path.join(abs_dirs['abs_work_dir'], 'testdata'),
            'gaia_l10n_base_dir': os.path.join(abs_dirs['abs_work_dir'], 'gaia-l10n'),
            'compare_locales_dir': os.path.join(abs_dirs['abs_work_dir'], 'compare-locales'),
        }

        abs_dirs.update(dirs)
        self.abs_dirs = abs_dirs
        return self.abs_dirs

    def query_gecko_config_path(self):
        conf_file = self.config.get('gecko_config')
        if conf_file is None:
            conf_file = os.path.join(
                'b2g', 'config',
                self.config.get('b2g_config_dir', self.config['target']),
                'config.json'
            )
        return conf_file

    def load_gecko_config(self):
        if self.gecko_config:
            return self.gecko_config

        if 'gecko_config' not in self.config:
            # Grab from the remote if we're not overriding on the cmdline
            self.gecko_config = self.query_remote_gecko_config()
            return self.gecko_config

        dirs = self.query_abs_dirs()
        conf_file = self.query_gecko_config_path()
        if not os.path.isabs(conf_file):
            conf_file = os.path.abspath(os.path.join(dirs['src'], conf_file))

        if os.path.exists(conf_file):
            self.info("gecko_config file: %s" % conf_file)
            self.run_command(['cat', conf_file])
            self.gecko_config = json.load(open(conf_file))
            return self.gecko_config

        # The file doesn't exist; let's try loading it remotely
        self.gecko_config = self.query_remote_gecko_config()
        return self.gecko_config

    def query_repo(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return 'http://hg.mozilla.org/%s' % self.buildbot_config['properties']['repo_path']
        else:
            return self.config['repo']

    def query_branch(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['branch']
        else:
            return os.path.basename(self.query_repo())

    def query_buildid(self):
        if self.buildid:
            return self.buildid
        dirs = self.query_abs_dirs()
        platform_ini = os.path.join(dirs['work_dir'], 'out', 'target',
                                    'product', self.config['target'], 'system',
                                    'b2g', 'platform.ini')
        data = self.read_from_file(platform_ini)
        buildid = re.search("^BuildID=(\d+)$", data, re.M)
        if buildid:
            self.buildid = buildid.group(1)
            return self.buildid

    def query_version(self):
        data = self.read_from_file(self.application_ini)
        version = re.search("^Version=(.+)$", data, re.M)
        if version:
            return version.group(1)

    def query_revision(self):
        if 'revision' in self.buildbot_properties:
            return self.buildbot_properties['revision']

        if self.buildbot_config and 'sourcestamp' in self.buildbot_config:
            return self.buildbot_config['sourcestamp']['revision']

        return None

    def query_translated_revision(self, url, project, rev, attempts=30, sleeptime=30,
                                  project_name=None):
        if project_name is None:
            project_name = project
        url = '%s/%s/git/%s' % (url, project, rev)
        self.info('Mapping %s revision from hg to git using %s' % (project_name, url))
        n = 1
        while n <= attempts:
            try:
                r = urllib2.urlopen(url, timeout=10)
                j = json.loads(r.readline())
                if j['git_rev'] is None:
                    # allow for user repos in staging
                    if self.config.get("require_git_rev", True):
                        raise Exception("Mapper returned a git revision of None; maybe it needs more time.")
                    else:
                        self.warning("Mapper returned a git revision of None.  Accepting because require_git_rev is False.")
                return j['git_rev']
            except Exception, err:
                self.warning('Error: %s' % str(err))
                if n == attempts:
                    self.fatal('Giving up on %s git revision for %s.' % (project_name, rev))
                if sleeptime > 0:
                    self.info('Sleeping %i seconds before retrying' % sleeptime)
                    time.sleep(sleeptime)
                continue
            finally:
                n += 1

    def get_hg_commit_time(self, repo_dir, rev):
        """Returns the commit time for given `rev` in unix epoch time"""
        hg = self.query_exe('hg')
        cmd = [
            hg,
            'log',
            '-R', repo_dir,
            '-r', rev,
            '--template', '{date|hgdate}'
        ]
        try:
            # {date|hgdate} returns a space-separated tuple of unixtime,
            # timezone offset
            output = self.get_output_from_command(cmd)
        except Exception:
            # Failed to run hg for some reason
            self.exception("failed to run hg log; using timestamp of 0 instead", level=WARNING)
            return 0

        try:
            t = output.split()[0]
            return int(t)
        except (ValueError, IndexError):
            self.exception("failed to parse hg log output; using timestamp of 0 instead", level=WARNING)
            return 0

    def query_do_upload(self):
        # always upload nightlies, but not dep builds for some platforms
        if self.query_is_nightly():
            return True
        if self.config['target'] in self.config.get('upload_dep_target_exclusions', []):
            return False
        return True

    def query_build_env(self):
        """Retrieves the environment for building"""
        dirs = self.query_abs_dirs()
        gecko_config = self.load_gecko_config()
        env = self.query_env()
        for k, v in gecko_config.get('env', {}).items():
            v = v.format(workdir=dirs['abs_work_dir'],
                         srcdir=os.path.abspath(dirs['src']))
            env[k] = v
        if self.config.get('variant'):
            v = str(self.config['variant'])
            env['VARIANT'] = v
        if self.config.get('ccache'):
            env['CCACHE_BASEDIR'] = dirs['work_dir']
        # If we get a buildid from buildbot, pass that in as MOZ_BUILD_DATE
        if self.buildbot_config and 'buildid' in self.buildbot_config.get('properties', {}):
            env['MOZ_BUILD_DATE'] = self.buildbot_config['properties']['buildid']

        if 'B2G_UPDATE_CHANNEL' not in env:
            env['B2G_UPDATE_CHANNEL'] = self.config['update_channel']

        return env

    def query_hgweb_url(self, repo, rev, filename=None):
        if filename:
            url = "{baseurl}/raw-file/{rev}/{filename}".format(
                baseurl=repo,
                rev=rev,
                filename=filename)
        else:
            url = "{baseurl}/rev/{rev}".format(
                baseurl=repo,
                rev=rev)
        return url

    def query_gitweb_url(self, repo, rev, filename=None):
        bits = urlparse.urlparse(repo)
        repo = bits.path.lstrip('/')
        if filename:
            url = "{scheme}://{host}/?p={repo};a=blob;f={filename};h={rev}".format(
                scheme=bits.scheme,
                host=bits.netloc,
                repo=repo,
                filename=filename,
                rev=rev)
        else:
            url = "{scheme}://{host}/?p={repo};a=tree;h={rev}".format(
                scheme=bits.scheme,
                host=bits.netloc,
                repo=repo,
                rev=rev)
        return url

    def query_remote_gecko_config(self):
        repo = self.query_repo()
        # TODO: Hardcoding this sucks
        if 'hg.mozilla.org' in repo:
            rev = self.query_revision()
            if rev is None:
                rev = 'default'

            config_path = self.query_gecko_config_path()
            # Handle local files vs. in-repo files
            url = self.query_hgweb_url(repo, rev, config_path)
            return self.retry(self.load_json_from_url, args=(url,))

    # Actions {{{2
    def clobber(self):
        dirs = self.query_abs_dirs()
        PurgeMixin.clobber(
            self,
            always_clobber_dirs=[
                dirs['abs_upload_dir'],
                dirs['testdata_dir'],
            ],
        )

    def checkout_sources(self):
        dirs = self.query_abs_dirs()
        gecko_config = self.load_gecko_config()
        if gecko_config.get('config_version') >= 2:
            # New behaviour!
            # Get B2G, b2g-manifest
            b2g_manifest_branch = gecko_config.get('b2g_manifest_branch', 'master')
            repos = [
                {'vcs': 'gittool', 'repo': 'https://git.mozilla.org/b2g/B2G.git', 'dest': dirs['work_dir']},
                {'vcs': 'gittool', 'repo': 'https://git.mozilla.org/b2g/b2g-manifest.git', 'dest': os.path.join(dirs['work_dir'], 'b2g-manifest'), 'branch': b2g_manifest_branch},
            ]
            self.vcs_checkout_repos(repos)

            # Now munge the manifest
            manifest_filename = gecko_config.get('b2g_manifest', self.config['target'] + '.xml')
            manifest_filename = os.path.join(dirs['work_dir'], 'b2g-manifest', manifest_filename)
            manifest = load_manifest(manifest_filename)

            mapping_func = functools.partial(map_remote, mappings=self.config['repo_remote_mappings'])

            rewrite_remotes(manifest, mapping_func)
            # Remove gecko, since we'll be checking that out ourselves
            gecko_node = remove_project(manifest, 'gecko.git')
            if not gecko_node:
                self.fatal("couldn't remove gecko from manifest")

            # repo requires a git repo to work with
            # so we create a temporary repo, put our manifest file into it, and
            # commit it
            manifest_dir = os.path.join(dirs['work_dir'], 'tmp_manifest')
            self.rmtree(manifest_dir)
            self.mkdir_p(manifest_dir)

            manifest_filename = os.path.join(manifest_dir, self.config['target'] + '.xml')
            manifest_file = open(manifest_filename, 'w')
            manifest.writexml(manifest_file)
            manifest_file.close()

            git = self.query_exe('git')
            self.run_command([git, 'init'], cwd=manifest_dir, halt_on_failure=True)
            self.run_command([git, 'add', manifest_filename], cwd=manifest_dir, halt_on_failure=True)
            self.run_command([git, 'commit', '-m', 'manifest'], cwd=manifest_dir, halt_on_failure=True)
            self.run_command([git, 'branch', '-m', b2g_manifest_branch], cwd=manifest_dir, halt_on_failure=True)

            # We need to reset gaia, since the build process locally modifies
            # files here, which can break sync later. \o/
            gaia_dir = os.path.join(dirs['work_dir'], 'gaia')
            if os.path.exists(gaia_dir):
                self.run_command([git, 'reset', '--hard'], cwd=gaia_dir)

            repo_repo = self.config['repo_repo']

            # Check it out!
            if 'repo_mirror_dir' in self.config:
                repo_mirror_dir = self.config['repo_mirror_dir']
                self.mkdir_p(repo_mirror_dir)
                # Blow away .repo and start from scratch. This sucks.
                # TODO: Can we avoid this?
                self.rmtree(os.path.join(repo_mirror_dir, '.repo'))
                repo = os.path.join(dirs['work_dir'], 'repo')
                self.run_command([repo, "init", "--repo-url", repo_repo, "-q", "--mirror", "-u", manifest_dir, "-m", self.config['target'] + '.xml', '-b', b2g_manifest_branch], cwd=repo_mirror_dir, halt_on_failure=True)
                self.run_command([repo, "sync", "--quiet"], cwd=repo_mirror_dir, halt_on_failure=True)

                # Now check it out into our local working directory
                self.run_command([repo, "init", "--repo-url", repo_repo, "-q", "--reference", repo_mirror_dir, "-u", manifest_dir, "-m", self.config['target'] + '.xml', '-b', b2g_manifest_branch], cwd=dirs['work_dir'], halt_on_failure=True)
            else:
                # Non-mirror mode
                self.run_command([repo, "init", "--repo-url", repo_repo, "-q", "-u", manifest_dir, "-m", self.config['target'] + '.xml', '-b', b2g_manifest_branch], cwd=dirs['work_dir'], halt_on_failure=True)
            self.run_command([repo, "sync", "--quiet"], cwd=dirs['work_dir'], halt_on_failure=True)

            # output our sources.xml, make a copy for update_sources_xml()
            self.run_command(["./gonk-misc/add-revision.py", "-o", "sources.xml", "--force", ".repo/manifest.xml"], cwd=dirs["work_dir"], halt_on_failure=True)
            self.run_command(["cat", "sources.xml"], cwd=dirs['work_dir'], halt_on_failure=True)
            self.run_command(["cp", "-p", "sources.xml", "sources.xml.original"], cwd=dirs['work_dir'], halt_on_failure=True)
            manifest = load_manifest(os.path.join(dirs['work_dir'], 'sources.xml'))
            gaia_node = get_project(manifest, "gaia.git")
            gaia_rev = gaia_node.getAttribute("revision")
            gaia_remote = get_remote(manifest, gaia_node.getAttribute('remote'))
            gaia_repo = "%s/%s" % (gaia_remote.getAttribute('fetch'), gaia_node.getAttribute('name'))
            gaia_url = self.query_gitweb_url(gaia_repo, gaia_rev)
            self.set_buildbot_property("gaia_revision", gaia_rev, write_to_file=True)
            self.info("TinderboxPrint: gaia_revlink: %s" % gaia_url)

            # Now we can checkout gecko and other stuff
            self.checkout_gecko()
            self.checkout_gecko_l10n()
            self.checkout_gaia_l10n()
            self.checkout_compare_locales()
            return

        # Old behaviour
        self.checkout_gecko()
        self.checkout_gaia()
        self.checkout_gaia_l10n()
        self.checkout_gecko_l10n()
        self.checkout_compare_locales()

    def get_blobs(self):
        self.download_blobs()
        self.unpack_blobs()

    def write_b2g_config(self):
        if self.wrote_b2g_config:
            return

        dirs = self.query_abs_dirs()

        # Write .config to point to the correct object directory for gecko
        # and specifies which target to use
        device_name = self.config['target']
        device = {
            'emulator': 'generic',
            'pandaboard': 'panda',
        }.get(device_name, device_name)

        # TODO: this re-implements some config.sh logic
        # TODO: write to .userconfig if we're using snapshots
        lines = [
            "GECKO_OBJDIR=%s" % self.objdir,
            "DEVICE_NAME=%s" % device_name,
            "DEVICE=%s" % device,
        ]
        # TODO: eh? what's this for? config.sh does it, but why?
        if device_name == 'emulator':
            lines.append("LUNCH=full-eng")

        # Make sure we get a blank line at the end
        lines.append("")

        self.write_to_file(
            os.path.join(dirs['work_dir'], '.config'),
            "\n".join(lines)
        )

        self.wrote_b2g_config = True

    def checkout_gecko(self):
        '''
        If you want a different revision of gecko to be used you can use the
        --checkout-revision flag. This is necessary for trees that are not
        triggered by a gecko commit but by an external tree (like gaia).
        '''
        dirs = self.query_abs_dirs()

        # Make sure the parent directory to gecko exists so that 'hg share ...
        # build/gecko' works
        self.mkdir_p(os.path.dirname(dirs['src']))

        repo = self.query_repo()
        if "checkout_revision" in self.config:
            rev = self.vcs_checkout(repo=repo, dest=dirs['src'], revision=self.config["checkout_revision"])
            # in this case, self.query_revision() will be returning the "revision" that triggered the job
            # we know that it is not a gecko revision that did so
            self.set_buildbot_property('revision', self.query_revision(), write_to_file=True)
        else:
            # a gecko revision triggered this job; self.query_revision() will return it
            rev = self.vcs_checkout(repo=repo, dest=dirs['src'], revision=self.query_revision())
            self.set_buildbot_property('revision', rev, write_to_file=True)
        self.set_buildbot_property('gecko_revision', rev, write_to_file=True)

    def download_blobs(self):
        c = self.config
        dirs = self.query_abs_dirs()
        gonk_url = None
        if 'gonk_snapshot_url' in c:
            # We've overridden which gonk to use
            gonk_url = c['gonk_snapshot_url']
        else:
            gecko_config = self.load_gecko_config()
            if 'tooltool_manifest' in gecko_config:
                # The manifest is relative to the gecko config
                config_dir = os.path.join(dirs['src'], 'b2g', 'config',
                                          self.config.get('b2g_config_dir', self.config['target']))
                manifest = os.path.abspath(os.path.join(config_dir, gecko_config['tooltool_manifest']))
                self.tooltool_fetch(manifest, dirs['work_dir'])
                return
            gonk_url = gecko_config['gonk_snapshot_url']

        if gonk_url:
            if os.path.exists("gonk.tar.xz"):
                self.info("Skipping download of %s because we have a local copy already" % gonk_url)
            else:
                retval = self.download_file(gonk_url, os.path.join(dirs['work_dir'], 'gonk.tar.xz'))
                if retval is None:
                    self.fatal("failed to download gonk", exit_code=2)

    def unpack_blobs(self):
        dirs = self.query_abs_dirs()
        tar = self.query_exe('tar', return_type="list")
        gonk_snapshot = os.path.join(dirs['abs_work_dir'], 'gonk.tar.xz')
        if os.path.exists(gonk_snapshot):
            mtime = int(os.path.getmtime(gonk_snapshot))
            mtime_file = os.path.join(dirs['abs_work_dir'], '.gonk_mtime')
            if os.path.exists(mtime_file):
                try:
                    prev_mtime = int(self.read_from_file(mtime_file, error_level=INFO))
                    if mtime == prev_mtime:
                        # transition code - help existing build dirs without a sources.xml.original
                        sourcesfile = os.path.join(dirs['work_dir'], 'sources.xml')
                        sourcesfile_orig = sourcesfile + '.original'
                        if not os.path.exists(sourcesfile_orig):
                            self.run_command(tar + ["xf", "gonk.tar.xz", "--strip-components", "1", "B2G_default_*/sources.xml"],
                                             cwd=dirs['work_dir'])
                            self.run_command(["cp", "-p", sourcesfile, sourcesfile_orig], cwd=dirs['work_dir'])
                        # end transition code
                        self.info("We already have this gonk unpacked; skipping")
                        return
                except:
                    pass

            self.info("Writing %s to %s" % (mtime, mtime_file))
            self.write_to_file(mtime_file, str(mtime))
            retval = self.run_command(tar + ["xf", "gonk.tar.xz", "--strip-components", "1"], cwd=dirs['work_dir'])
            if retval != 0:
                self.fatal("failed to unpack gonk", exit_code=2)

            # output our sources.xml, make a copy for update_sources_xml()
            self.run_command(["cat", "sources.xml"], cwd=dirs['work_dir'])
            self.run_command(["cp", "-p", "sources.xml", "sources.xml.original"], cwd=dirs['work_dir'])

        gecko_config = self.load_gecko_config()
        extra_tarballs = self.config.get('additional_source_tarballs', [])
        if 'additional_source_tarballs' in gecko_config:
            extra_tarballs.extend(gecko_config['additional_source_tarballs'])

        for tarball in extra_tarballs:
            self.run_command(tar + ["xf", tarball], cwd=dirs['work_dir'],
                             halt_on_failure=True)

    def checkout_gaia(self):
        dirs = self.query_abs_dirs()
        gecko_config = self.load_gecko_config()
        gaia_config = gecko_config.get('gaia')
        if gaia_config:
            dest = os.path.join(dirs['abs_work_dir'], 'gaia')
            repo = gaia_config['repo']
            branch = gaia_config.get('branch')
            vcs = gaia_config['vcs']
            rev = self.vcs_checkout(repo=repo, dest=dest, vcs=vcs, branch=branch)
            self.set_buildbot_property('gaia_revision', rev, write_to_file=True)
            self.info("TinderboxPrint: gaia_revlink: %s/rev/%s" % (repo, rev))

    def checkout_gaia_l10n(self):
        if not self.config.get('gaia_languages_file'):
            self.info('Skipping checkout_gaia_l10n because no gaia language file was specified.')
            return

        l10n_config = self.load_gecko_config().get('gaia', {}).get('l10n')
        if not l10n_config:
            self.fatal("gaia.l10n is required in the gecko config when --gaia-languages-file is specified.")

        abs_work_dir = self.query_abs_dirs()['abs_work_dir']
        languages_file = os.path.join(abs_work_dir, 'gaia', self.config['gaia_languages_file'])
        l10n_base_dir = self.query_abs_dirs()['gaia_l10n_base_dir']

        self.pull_gaia_locale_source(l10n_config, parse_config_file(languages_file).keys(), l10n_base_dir)

    def checkout_gecko_l10n(self):
        hg_l10n_base = self.load_gecko_config().get('gecko_l10n_root')
        self.pull_locale_source(hg_l10n_base=hg_l10n_base)
        gecko_locales = self.query_locales()
        # populate b2g/overrides, which isn't in gecko atm
        dirs = self.query_abs_dirs()
        for locale in gecko_locales:
            self.mkdir_p(os.path.join(dirs['abs_l10n_dir'], locale, 'b2g', 'chrome', 'overrides'))
            self.copytree(os.path.join(dirs['abs_l10n_dir'], locale, 'mobile', 'overrides'),
                          os.path.join(dirs['abs_l10n_dir'], locale, 'b2g', 'chrome', 'overrides'),
                          error_level=FATAL)

    def checkout_compare_locales(self):
        dirs = self.query_abs_dirs()
        dest = dirs['compare_locales_dir']
        repo = self.config['compare_locales_repo']
        rev = self.config['compare_locales_rev']
        vcs = self.config['compare_locales_vcs']
        abs_rev = self.vcs_checkout(repo=repo, dest=dest, revision=rev, vcs=vcs)
        self.set_buildbot_property('compare_locales_revision', abs_rev, write_to_file=True)

    def query_do_translate_hg_to_git(self, gecko_config_key=None):
        manifest_config = self.config.get('manifest', {})
        branch = self.query_branch()
        if self.query_is_nightly() and branch in manifest_config['branches'] and \
                manifest_config.get('translate_hg_to_git'):
            if gecko_config_key is None:
                return True
            if self.gecko_config.get(gecko_config_key):
                return True
        return False

    def _generate_git_locale_manifest(self, locale, url, git_repo,
                                      revision, git_base_url, local_path):
        l10n_git_sha = self.query_translated_revision(url, 'l10n', revision, project_name="l10n %s" % locale)
        return '  <project name="%s" path="%s" remote="mozillaorg" revision="%s"/>' % (git_repo.replace(git_base_url, ''), local_path, l10n_git_sha)

    def _generate_locale_manifest(self, git_base_url="https://git.mozilla.org/release/"):
        """ Add the locales to the source manifest.
        """
        manifest_config = self.config.get('manifest', {})
        locale_manifest = []
        if self.gaia_locale_revisions:
            gaia_l10n_git_root = None
            if self.query_do_translate_hg_to_git(gecko_config_key='gaia_l10n_git_root'):
                gaia_l10n_git_root = self.gecko_config['gaia_l10n_git_root']
            for locale in self.gaia_locale_revisions.keys():
                repo = self.gaia_locale_revisions[locale]['repo']
                revision = self.gaia_locale_revisions[locale]['revision']
                locale_manifest.append('  <!-- Mercurial-Information: <project name="%s" path="gaia-l10n/%s" remote="hgmozillaorg" revision="%s"/> -->' %
                                       (repo.replace('http://hg.mozilla.org/', ''), locale, revision))
                if gaia_l10n_git_root:
                    locale_manifest.append(
                        self._generate_git_locale_manifest(
                            locale,
                            manifest_config['translate_base_url'],
                            gaia_l10n_git_root % {'locale': locale},
                            revision,
                            git_base_url,
                            "gaia-l10n/%s" % locale,
                        )
                    )
        if self.gecko_locale_revisions:
            gecko_l10n_git_root = None
            if self.query_do_translate_hg_to_git(gecko_config_key='gecko_l10n_git_root'):
                gecko_l10n_git_root = self.gecko_config['gecko_l10n_git_root']
            for locale in self.gecko_locale_revisions.keys():
                repo = self.gecko_locale_revisions[locale]['repo']
                revision = self.gecko_locale_revisions[locale]['revision']
                locale_manifest.append('  <!-- Mercurial-Information: <project name="%s" path="gecko-l10n/%s" remote="hgmozillaorg" revision="%s"/> -->' %
                                       (repo.replace('http://hg.mozilla.org/', ''), locale, revision))
                if gecko_l10n_git_root:
                    locale_manifest.append(
                        self._generate_git_locale_manifest(
                            locale,
                            manifest_config['translate_base_url'],
                            gecko_l10n_git_root % {'locale': locale},
                            revision,
                            git_base_url,
                            "gecko-l10n/%s" % locale,
                        )
                    )
        return locale_manifest

    def update_source_manifest(self):
        dirs = self.query_abs_dirs()
        gecko_config = self.load_gecko_config()
        gaia_config = gecko_config.get('gaia')
        manifest_config = self.config.get('manifest', {})

        sourcesfile = os.path.join(dirs['work_dir'], 'sources.xml')
        sourcesfile_orig = sourcesfile + '.original'
        sources = self.read_from_file(sourcesfile_orig, verbose=False)
        dom = xml.dom.minidom.parseString(sources)
        git_base_url = "https://git.mozilla.org/"
        for element in dom.getElementsByTagName('remote'):
            if element.getAttribute('name') == 'mozillaorg':
                pieces = urlparse.urlparse(element.getAttribute('fetch'))
                if pieces:
                    git_base_url = "https://git.mozilla.org%s" % pieces[2]
                    if not git_base_url.endswith('/'):
                        git_base_url += "/"
                    self.info("Found git_base_url of %s in manifest." % git_base_url)
                    break
        else:
            self.warning("Couldn't find git_base_url in manifest; using %s" % git_base_url)
        new_sources = []

        for line in sources.splitlines():
            new_sources.append(line)
            if 'Gonk specific things' in line:
                new_sources.append('  <!-- Mercurial-Information: <remote fetch="http://hg.mozilla.org/" name="hgmozillaorg"> -->')
                new_sources.append('  <!-- Mercurial-Information: <project name="%s" path="gecko" remote="hgmozillaorg" revision="%s"/> -->' %
                                   (self.buildbot_config['properties']['repo_path'], self.buildbot_properties['gecko_revision']))
                if 'gaia_revision' in self.buildbot_properties:
                    new_sources.append('  <!-- Mercurial-Information: <project name="%s" path="gaia" remote="hgmozillaorg" revision="%s"/> -->' %
                                       (gaia_config['repo'].replace('http://hg.mozilla.org/', ''), self.buildbot_properties['gaia_revision']))

                if self.query_do_translate_hg_to_git():
                    url = manifest_config['translate_base_url']
                    gecko_git = self.query_translated_revision(url, 'gecko', self.buildbot_properties['gecko_revision'])
                    new_sources.append('  <project name="%s" path="gecko" remote="mozillaorg" revision="%s"/>' % ("https://git.mozilla.org/releases/gecko.git".replace(git_base_url, ''), gecko_git))
                    if 'gaia_revision' in self.buildbot_properties:
                        gaia_git = self.query_translated_revision(url, 'gaia', self.buildbot_properties['gaia_revision'])
                        new_sources.append('  <project name="%s" path="gaia" remote="mozillaorg" revision="%s"/>' % ("https://git.mozilla.org/releases/gaia.git".replace(git_base_url, ''), gaia_git))

                    # Figure out when our gaia commit happened
                    gaia_time = self.get_hg_commit_time(os.path.join(dirs['abs_work_dir'], 'gaia'), self.buildbot_properties['gaia_revision'])

                    # Write the gaia commit information to 'gaia_commit_override.txt'
                    gaia_override = os.path.join(dirs['abs_work_dir'], 'gaia', 'gaia_commit_override.txt')
                    self.write_to_file(gaia_override, "%s\n%s\n" % (gaia_git, gaia_time))
                new_sources.extend(self._generate_locale_manifest(git_base_url=git_base_url))

        self.write_to_file(sourcesfile, "\n".join(new_sources), verbose=False)
        self.run_command(["diff", "-u", sourcesfile_orig, sourcesfile], success_codes=[1])

    def build(self):
        dirs = self.query_abs_dirs()
        gecko_config = self.load_gecko_config()
        build_targets = gecko_config.get('build_targets', [])
        cmd = ['./build.sh'] + build_targets
        env = self.query_build_env()
        if self.config.get('gaia_languages_file'):
            env['LOCALE_BASEDIR'] = dirs['gaia_l10n_base_dir']
            env['LOCALES_FILE'] = os.path.join(dirs['abs_work_dir'], 'gaia', self.config['gaia_languages_file'])<commit_msg>Bug 838321: We don't need to use mapper to find git revision for gaia if we pulled gaia from git. r=rail<commit_after>#!/usr/bin/env python
# Mozilla licence shtuff

import sys
import os
import glob
import re
import tempfile
from datetime import datetime
import urllib2
import urlparse
import time
import xml.dom.minidom
import functools

try:
    import simplejson as json
    assert json
except ImportError:
    import json

# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))

# import the guts
from mozharness.base.config import parse_config_file
from mozharness.base.script import BaseScript
from mozharness.base.vcs.vcsbase import VCSMixin
from mozharness.base.transfer import TransferMixin
from mozharness.base.errors import MakefileErrorList
from mozharness.base.log import WARNING, ERROR, INFO, FATAL
from mozharness.mozilla.l10n.locales import GaiaLocalesMixin, LocalesMixin
from mozharness.mozilla.mock import MockMixin
from mozharness.mozilla.tooltool import TooltoolMixin
from mozharness.mozilla.buildbot import BuildbotMixin
from mozharness.mozilla.purge import PurgeMixin
from mozharness.mozilla.signing import SigningMixin
from mozharness.mozilla.repo_manifest import (load_manifest, rewrite_remotes,
                                              remove_project, get_project, get_remote)

# B2G builds complain about java...but it doesn't seem to be a problem
# Let's turn those into WARNINGS instead
B2GMakefileErrorList = MakefileErrorList + [
    {'substr': r'''NS_ERROR_FILE_ALREADY_EXISTS: Component returned failure code''', 'level': ERROR},
]
B2GMakefileErrorList.insert(0, {'substr': r'/bin/bash: java: command not found', 'level': WARNING})


def map_remote(r, mappings):
    """
    Helper function for mapping git remotes
    """
    remote = r.getAttribute('fetch')
    if remote in mappings:
        r.setAttribute('fetch', mappings[remote])
        return r
    return None


class B2GBuild(LocalesMixin, MockMixin, PurgeMixin, BaseScript, VCSMixin, TooltoolMixin,
               TransferMixin, BuildbotMixin, GaiaLocalesMixin, SigningMixin):
    config_options = [
        [["--repo"], {
            "dest": "repo",
            "help": "which gecko repo to check out",
        }],
        [["--gonk-snapshot"], {
            "dest": "gonk_snapshot_url",
            "help": "override the gonk snapshot specified in the build config",
        }],
        [["--target"], {
            "dest": "target",
            "help": "specify which build type to do",
        }],
        [["--b2g-config-dir"], {
            "dest": "b2g_config_dir",
            "help": "specify which in-tree config directory to use, relative to b2g/config/ (defaults to --target)",
        }],
        [["--gecko-config"], {
            "dest": "gecko_config",
            "help": "specfiy alternate location for gecko config",
        }],
        [["--disable-ccache"], {
            "dest": "ccache",
            "action": "store_false",
            "help": "disable ccache",
        }],
        [["--gaia-languages-file"], {
            "dest": "gaia_languages_file",
            "help": "languages file for gaia multilocale profile",
        }],
        [["--gecko-languages-file"], {
            "dest": "locales_file",
            "help": "languages file for gecko multilocale",
        }],
        [["--gecko-l10n-base-dir"], {
            "dest": "l10n_dir",
            "help": "dir to clone gecko l10n repos into, relative to the work directory",
        }],
        [["--merge-locales"], {
            "dest": "merge_locales",
            "help": "Dummy option to keep from burning. We now always merge",
        }],
        [["--variant"], {
            "dest": "variant",
            "help": "b2g build variant. overrides gecko config's value",
        }],
        [["--checkout-revision"], {
            "dest": "checkout_revision",
            "help": "checkout a specific gecko revision.",
        }],
        [["--additional-source-tarballs"], {
            "action": "extend",
            "type": "string",
            "dest": "additional_source_tarballs",
            "help": "Additional source tarballs to extract",
        }],
        [["--update-channel"], {
            "dest": "update_channel",
            "help": "b2g update channel",
        }],
    ]

    def __init__(self, require_config_file=False):
        self.gecko_config = None
        self.buildid = None
        self.wrote_b2g_config = False
        LocalesMixin.__init__(self)
        BaseScript.__init__(self,
                            config_options=self.config_options,
                            all_actions=[
                                'clobber',
                                'checkout-sources',
                                # Deprecated
                                'checkout-gecko',
                                'download-gonk',
                                'unpack-gonk',
                                'checkout-gaia',
                                'checkout-gaia-l10n',
                                'checkout-gecko-l10n',
                                'checkout-compare-locales',
                                # End deprecated
                                'get-blobs',
                                'update-source-manifest',
                                'build',
                                'build-symbols',
                                'make-updates',
                                'build-update-testdata',
                                'prep-upload',
                                'upload',
                                'make-update-xml',
                                'upload-updates',
                                'make-socorro-json',
                                'upload-source-manifest',
                            ],
                            default_actions=[
                                'checkout-sources',
                                'get-blobs',
                                'build',
                            ],
                            require_config_file=require_config_file,

                            # Default configuration
                            config={
                                'default_vcs': 'hgtool',
                                'vcs_share_base': os.environ.get('HG_SHARE_BASE_DIR'),
                                'ccache': True,
                                'buildbot_json_path': os.environ.get('PROPERTIES_FILE'),
                                'tooltool_servers': None,
                                'ssh_key': None,
                                'ssh_user': None,
                                'upload_remote_host': None,
                                'upload_remote_basepath': None,
                                'enable_try_uploads': False,
                                'tools_repo': 'http://hg.mozilla.org/build/tools',
                                'locales_dir': 'gecko/b2g/locales',
                                'l10n_dir': 'gecko-l10n',
                                'ignore_locales': ['en-US', 'multi'],
                                'locales_file': 'gecko/b2g/locales/all-locales',
                                'mozilla_dir': 'build/gecko',
                                'objdir': 'build/objdir-gecko',
                                'merge_locales': True,
                                'compare_locales_repo': 'http://hg.mozilla.org/build/compare-locales',
                                'compare_locales_rev': 'RELEASE_AUTOMATION',
                                'compare_locales_vcs': 'hgtool',
                                'repo_repo': "https://git.mozilla.org/external/google/gerrit/git-repo.git",
                                'repo_remote_mappings': {},
                                'update_channel': 'nightly',
                            },
                            )

        dirs = self.query_abs_dirs()
        self.objdir = os.path.join(dirs['work_dir'], 'objdir-gecko')
        self.marfile = "%s/dist/b2g-update/b2g-gecko-update.mar" % self.objdir
        self.application_ini = os.path.join(
            dirs['work_dir'], 'out', 'target', 'product',
            self.config['target'], 'system', 'b2g', 'application.ini')

    def _pre_config_lock(self, rw_config):
        super(B2GBuild, self)._pre_config_lock(rw_config)

        if self.buildbot_config is None:
            self.info("Reading buildbot build properties...")
            self.read_buildbot_config()

        if 'target' not in self.config:
            self.fatal("Must specify --target!")

        if not (self.buildbot_config and 'properties' in self.buildbot_config) and 'repo' not in self.config:
            self.fatal("Must specify --repo")

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = LocalesMixin.query_abs_dirs(self)

        c = self.config
        dirs = {
            'src': os.path.join(c['work_dir'], 'gecko'),
            'work_dir': abs_dirs['abs_work_dir'],
            'testdata_dir': os.path.join(abs_dirs['abs_work_dir'], 'testdata'),
            'gaia_l10n_base_dir': os.path.join(abs_dirs['abs_work_dir'], 'gaia-l10n'),
            'compare_locales_dir': os.path.join(abs_dirs['abs_work_dir'], 'compare-locales'),
        }

        abs_dirs.update(dirs)
        self.abs_dirs = abs_dirs
        return self.abs_dirs

    def query_gecko_config_path(self):
        conf_file = self.config.get('gecko_config')
        if conf_file is None:
            conf_file = os.path.join(
                'b2g', 'config',
                self.config.get('b2g_config_dir', self.config['target']),
                'config.json'
            )
        return conf_file

    def load_gecko_config(self):
        if self.gecko_config:
            return self.gecko_config

        if 'gecko_config' not in self.config:
            # Grab from the remote if we're not overriding on the cmdline
            self.gecko_config = self.query_remote_gecko_config()
            return self.gecko_config

        dirs = self.query_abs_dirs()
        conf_file = self.query_gecko_config_path()
        if not os.path.isabs(conf_file):
            conf_file = os.path.abspath(os.path.join(dirs['src'], conf_file))

        if os.path.exists(conf_file):
            self.info("gecko_config file: %s" % conf_file)
            self.run_command(['cat', conf_file])
            self.gecko_config = json.load(open(conf_file))
            return self.gecko_config

        # The file doesn't exist; let's try loading it remotely
        self.gecko_config = self.query_remote_gecko_config()
        return self.gecko_config

    def query_repo(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return 'http://hg.mozilla.org/%s' % self.buildbot_config['properties']['repo_path']
        else:
            return self.config['repo']

    def query_branch(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['branch']
        else:
            return os.path.basename(self.query_repo())

    def query_buildid(self):
        if self.buildid:
            return self.buildid
        dirs = self.query_abs_dirs()
        platform_ini = os.path.join(dirs['work_dir'], 'out', 'target',
                                    'product', self.config['target'], 'system',
                                    'b2g', 'platform.ini')
        data = self.read_from_file(platform_ini)
        buildid = re.search("^BuildID=(\d+)$", data, re.M)
        if buildid:
            self.buildid = buildid.group(1)
            return self.buildid

    def query_version(self):
        data = self.read_from_file(self.application_ini)
        version = re.search("^Version=(.+)$", data, re.M)
        if version:
            return version.group(1)

    def query_revision(self):
        if 'revision' in self.buildbot_properties:
            return self.buildbot_properties['revision']

        if self.buildbot_config and 'sourcestamp' in self.buildbot_config:
            return self.buildbot_config['sourcestamp']['revision']

        return None

    def query_translated_revision(self, url, project, rev, attempts=30, sleeptime=30,
                                  project_name=None):
        if project_name is None:
            project_name = project
        url = '%s/%s/git/%s' % (url, project, rev)
        self.info('Mapping %s revision from hg to git using %s' % (project_name, url))
        n = 1
        while n <= attempts:
            try:
                r = urllib2.urlopen(url, timeout=10)
                j = json.loads(r.readline())
                if j['git_rev'] is None:
                    # allow for user repos in staging
                    if self.config.get("require_git_rev", True):
                        raise Exception("Mapper returned a git revision of None; maybe it needs more time.")
                    else:
                        self.warning("Mapper returned a git revision of None.  Accepting because require_git_rev is False.")
                return j['git_rev']
            except Exception, err:
                self.warning('Error: %s' % str(err))
                if n == attempts:
                    self.fatal('Giving up on %s git revision for %s.' % (project_name, rev))
                if sleeptime > 0:
                    self.info('Sleeping %i seconds before retrying' % sleeptime)
                    time.sleep(sleeptime)
                continue
            finally:
                n += 1

    def get_hg_commit_time(self, repo_dir, rev):
        """Returns the commit time for given `rev` in unix epoch time"""
        hg = self.query_exe('hg')
        cmd = [
            hg,
            'log',
            '-R', repo_dir,
            '-r', rev,
            '--template', '{date|hgdate}'
        ]
        try:
            # {date|hgdate} returns a space-separated tuple of unixtime,
            # timezone offset
            output = self.get_output_from_command(cmd)
        except Exception:
            # Failed to run hg for some reason
            self.exception("failed to run hg log; using timestamp of 0 instead", level=WARNING)
            return 0

        try:
            t = output.split()[0]
            return int(t)
        except (ValueError, IndexError):
            self.exception("failed to parse hg log output; using timestamp of 0 instead", level=WARNING)
            return 0

    def query_do_upload(self):
        # always upload nightlies, but not dep builds for some platforms
        if self.query_is_nightly():
            return True
        if self.config['target'] in self.config.get('upload_dep_target_exclusions', []):
            return False
        return True

    def query_build_env(self):
        """Retrieves the environment for building"""
        dirs = self.query_abs_dirs()
        gecko_config = self.load_gecko_config()
        env = self.query_env()
        for k, v in gecko_config.get('env', {}).items():
            v = v.format(workdir=dirs['abs_work_dir'],
                         srcdir=os.path.abspath(dirs['src']))
            env[k] = v
        if self.config.get('variant'):
            v = str(self.config['variant'])
            env['VARIANT'] = v
        if self.config.get('ccache'):
            env['CCACHE_BASEDIR'] = dirs['work_dir']
        # If we get a buildid from buildbot, pass that in as MOZ_BUILD_DATE
        if self.buildbot_config and 'buildid' in self.buildbot_config.get('properties', {}):
            env['MOZ_BUILD_DATE'] = self.buildbot_config['properties']['buildid']

        if 'B2G_UPDATE_CHANNEL' not in env:
            env['B2G_UPDATE_CHANNEL'] = self.config['update_channel']

        return env

    def query_hgweb_url(self, repo, rev, filename=None):
        if filename:
            url = "{baseurl}/raw-file/{rev}/{filename}".format(
                baseurl=repo,
                rev=rev,
                filename=filename)
        else:
            url = "{baseurl}/rev/{rev}".format(
                baseurl=repo,
                rev=rev)
        return url

    def query_gitweb_url(self, repo, rev, filename=None):
        bits = urlparse.urlparse(repo)
        repo = bits.path.lstrip('/')
        if filename:
            url = "{scheme}://{host}/?p={repo};a=blob;f={filename};h={rev}".format(
                scheme=bits.scheme,
                host=bits.netloc,
                repo=repo,
                filename=filename,
                rev=rev)
        else:
            url = "{scheme}://{host}/?p={repo};a=tree;h={rev}".format(
                scheme=bits.scheme,
                host=bits.netloc,
                repo=repo,
                rev=rev)
        return url

    def query_remote_gecko_config(self):
        repo = self.query_repo()
        # TODO: Hardcoding this sucks
        if 'hg.mozilla.org' in repo:
            rev = self.query_revision()
            if rev is None:
                rev = 'default'

            config_path = self.query_gecko_config_path()
            # Handle local files vs. in-repo files
            url = self.query_hgweb_url(repo, rev, config_path)
            return self.retry(self.load_json_from_url, args=(url,))

    # Actions {{{2
    def clobber(self):
        dirs = self.query_abs_dirs()
        PurgeMixin.clobber(
            self,
            always_clobber_dirs=[
                dirs['abs_upload_dir'],
                dirs['testdata_dir'],
            ],
        )

    def checkout_sources(self):
        dirs = self.query_abs_dirs()
        gecko_config = self.load_gecko_config()
        if gecko_config.get('config_version') >= 2:
            # New behaviour!
            # Get B2G, b2g-manifest
            b2g_manifest_branch = gecko_config.get('b2g_manifest_branch', 'master')
            repos = [
                {'vcs': 'gittool', 'repo': 'https://git.mozilla.org/b2g/B2G.git', 'dest': dirs['work_dir']},
                {'vcs': 'gittool', 'repo': 'https://git.mozilla.org/b2g/b2g-manifest.git', 'dest': os.path.join(dirs['work_dir'], 'b2g-manifest'), 'branch': b2g_manifest_branch},
            ]
            self.vcs_checkout_repos(repos)

            # Now munge the manifest
            manifest_filename = gecko_config.get('b2g_manifest', self.config['target'] + '.xml')
            manifest_filename = os.path.join(dirs['work_dir'], 'b2g-manifest', manifest_filename)
            manifest = load_manifest(manifest_filename)

            mapping_func = functools.partial(map_remote, mappings=self.config['repo_remote_mappings'])

            rewrite_remotes(manifest, mapping_func)
            # Remove gecko, since we'll be checking that out ourselves
            gecko_node = remove_project(manifest, 'gecko.git')
            if not gecko_node:
                self.fatal("couldn't remove gecko from manifest")

            # repo requires a git repo to work with
            # so we create a temporary repo, put our manifest file into it, and
            # commit it
            manifest_dir = os.path.join(dirs['work_dir'], 'tmp_manifest')
            self.rmtree(manifest_dir)
            self.mkdir_p(manifest_dir)

            manifest_filename = os.path.join(manifest_dir, self.config['target'] + '.xml')
            manifest_file = open(manifest_filename, 'w')
            manifest.writexml(manifest_file)
            manifest_file.close()

            git = self.query_exe('git')
            self.run_command([git, 'init'], cwd=manifest_dir, halt_on_failure=True)
            self.run_command([git, 'add', manifest_filename], cwd=manifest_dir, halt_on_failure=True)
            self.run_command([git, 'commit', '-m', 'manifest'], cwd=manifest_dir, halt_on_failure=True)
            self.run_command([git, 'branch', '-m', b2g_manifest_branch], cwd=manifest_dir, halt_on_failure=True)

            # We need to reset gaia, since the build process locally modifies
            # files here, which can break sync later. \o/
            gaia_dir = os.path.join(dirs['work_dir'], 'gaia')
            if os.path.exists(gaia_dir):
                self.run_command([git, 'reset', '--hard'], cwd=gaia_dir)

            repo_repo = self.config['repo_repo']

            # Check it out!
            if 'repo_mirror_dir' in self.config:
                repo_mirror_dir = self.config['repo_mirror_dir']
                self.mkdir_p(repo_mirror_dir)
                # Blow away .repo and start from scratch. This sucks.
                # TODO: Can we avoid this?
                self.rmtree(os.path.join(repo_mirror_dir, '.repo'))
                repo = os.path.join(dirs['work_dir'], 'repo')
                self.run_command([repo, "init", "--repo-url", repo_repo, "-q", "--mirror", "-u", manifest_dir, "-m", self.config['target'] + '.xml', '-b', b2g_manifest_branch], cwd=repo_mirror_dir, halt_on_failure=True)
                self.run_command([repo, "sync", "--quiet"], cwd=repo_mirror_dir, halt_on_failure=True)

                # Now check it out into our local working directory
                self.run_command([repo, "init", "--repo-url", repo_repo, "-q", "--reference", repo_mirror_dir, "-u", manifest_dir, "-m", self.config['target'] + '.xml', '-b', b2g_manifest_branch], cwd=dirs['work_dir'], halt_on_failure=True)
            else:
                # Non-mirror mode
                self.run_command([repo, "init", "--repo-url", repo_repo, "-q", "-u", manifest_dir, "-m", self.config['target'] + '.xml', '-b', b2g_manifest_branch], cwd=dirs['work_dir'], halt_on_failure=True)
            self.run_command([repo, "sync", "--quiet"], cwd=dirs['work_dir'], halt_on_failure=True)

            # output our sources.xml, make a copy for update_sources_xml()
            self.run_command(["./gonk-misc/add-revision.py", "-o", "sources.xml", "--force", ".repo/manifest.xml"], cwd=dirs["work_dir"], halt_on_failure=True)
            self.run_command(["cat", "sources.xml"], cwd=dirs['work_dir'], halt_on_failure=True)
            self.run_command(["cp", "-p", "sources.xml", "sources.xml.original"], cwd=dirs['work_dir'], halt_on_failure=True)
            manifest = load_manifest(os.path.join(dirs['work_dir'], 'sources.xml'))
            gaia_node = get_project(manifest, "gaia.git")
            gaia_rev = gaia_node.getAttribute("revision")
            gaia_remote = get_remote(manifest, gaia_node.getAttribute('remote'))
            gaia_repo = "%s/%s" % (gaia_remote.getAttribute('fetch'), gaia_node.getAttribute('name'))
            gaia_url = self.query_gitweb_url(gaia_repo, gaia_rev)
            self.set_buildbot_property("gaia_revision", gaia_rev, write_to_file=True)
            self.info("TinderboxPrint: gaia_revlink: %s" % gaia_url)

            # Now we can checkout gecko and other stuff
            self.checkout_gecko()
            self.checkout_gecko_l10n()
            self.checkout_gaia_l10n()
            self.checkout_compare_locales()
            return

        # Old behaviour
        self.checkout_gecko()
        self.checkout_gaia()
        self.checkout_gaia_l10n()
        self.checkout_gecko_l10n()
        self.checkout_compare_locales()

    def get_blobs(self):
        self.download_blobs()
        self.unpack_blobs()

    def write_b2g_config(self):
        if self.wrote_b2g_config:
            return

        dirs = self.query_abs_dirs()

        # Write .config to point to the correct object directory for gecko
        # and specifies which target to use
        device_name = self.config['target']
        device = {
            'emulator': 'generic',
            'pandaboard': 'panda',
        }.get(device_name, device_name)

        # TODO: this re-implements some config.sh logic
        # TODO: write to .userconfig if we're using snapshots
        lines = [
            "GECKO_OBJDIR=%s" % self.objdir,
            "DEVICE_NAME=%s" % device_name,
            "DEVICE=%s" % device,
        ]
        # TODO: eh? what's this for? config.sh does it, but why?
        if device_name == 'emulator':
            lines.append("LUNCH=full-eng")

        # Make sure we get a blank line at the end
        lines.append("")

        self.write_to_file(
            os.path.join(dirs['work_dir'], '.config'),
            "\n".join(lines)
        )

        self.wrote_b2g_config = True

    def checkout_gecko(self):
        '''
        If you want a different revision of gecko to be used you can use the
        --checkout-revision flag. This is necessary for trees that are not
        triggered by a gecko commit but by an external tree (like gaia).
        '''
        dirs = self.query_abs_dirs()

        # Make sure the parent directory to gecko exists so that 'hg share ...
        # build/gecko' works
        self.mkdir_p(os.path.dirname(dirs['src']))

        repo = self.query_repo()
        if "checkout_revision" in self.config:
            rev = self.vcs_checkout(repo=repo, dest=dirs['src'], revision=self.config["checkout_revision"])
            # in this case, self.query_revision() will be returning the "revision" that triggered the job
            # we know that it is not a gecko revision that did so
            self.set_buildbot_property('revision', self.query_revision(), write_to_file=True)
        else:
            # a gecko revision triggered this job; self.query_revision() will return it
            rev = self.vcs_checkout(repo=repo, dest=dirs['src'], revision=self.query_revision())
            self.set_buildbot_property('revision', rev, write_to_file=True)
        self.set_buildbot_property('gecko_revision', rev, write_to_file=True)

    def download_blobs(self):
        c = self.config
        dirs = self.query_abs_dirs()
        gonk_url = None
        if 'gonk_snapshot_url' in c:
            # We've overridden which gonk to use
            gonk_url = c['gonk_snapshot_url']
        else:
            gecko_config = self.load_gecko_config()
            if 'tooltool_manifest' in gecko_config:
                # The manifest is relative to the gecko config
                config_dir = os.path.join(dirs['src'], 'b2g', 'config',
                                          self.config.get('b2g_config_dir', self.config['target']))
                manifest = os.path.abspath(os.path.join(config_dir, gecko_config['tooltool_manifest']))
                self.tooltool_fetch(manifest, dirs['work_dir'])
                return
            gonk_url = gecko_config['gonk_snapshot_url']

        if gonk_url:
            if os.path.exists("gonk.tar.xz"):
                self.info("Skipping download of %s because we have a local copy already" % gonk_url)
            else:
                retval = self.download_file(gonk_url, os.path.join(dirs['work_dir'], 'gonk.tar.xz'))
                if retval is None:
                    self.fatal("failed to download gonk", exit_code=2)

    def unpack_blobs(self):
        dirs = self.query_abs_dirs()
        tar = self.query_exe('tar', return_type="list")
        gonk_snapshot = os.path.join(dirs['abs_work_dir'], 'gonk.tar.xz')
        if os.path.exists(gonk_snapshot):
            mtime = int(os.path.getmtime(gonk_snapshot))
            mtime_file = os.path.join(dirs['abs_work_dir'], '.gonk_mtime')
            if os.path.exists(mtime_file):
                try:
                    prev_mtime = int(self.read_from_file(mtime_file, error_level=INFO))
                    if mtime == prev_mtime:
                        # transition code - help existing build dirs without a sources.xml.original
                        sourcesfile = os.path.join(dirs['work_dir'], 'sources.xml')
                        sourcesfile_orig = sourcesfile + '.original'
                        if not os.path.exists(sourcesfile_orig):
                            self.run_command(tar + ["xf", "gonk.tar.xz", "--strip-components", "1", "B2G_default_*/sources.xml"],
                                             cwd=dirs['work_dir'])
                            self.run_command(["cp", "-p", sourcesfile, sourcesfile_orig], cwd=dirs['work_dir'])
                        # end transition code
                        self.info("We already have this gonk unpacked; skipping")
                        return
                except:
                    pass

            self.info("Writing %s to %s" % (mtime, mtime_file))
            self.write_to_file(mtime_file, str(mtime))
            retval = self.run_command(tar + ["xf", "gonk.tar.xz", "--strip-components", "1"], cwd=dirs['work_dir'])
            if retval != 0:
                self.fatal("failed to unpack gonk", exit_code=2)

            # output our sources.xml, make a copy for update_sources_xml()
            self.run_command(["cat", "sources.xml"], cwd=dirs['work_dir'])
            self.run_command(["cp", "-p", "sources.xml", "sources.xml.original"], cwd=dirs['work_dir'])

        gecko_config = self.load_gecko_config()
        extra_tarballs = self.config.get('additional_source_tarballs', [])
        if 'additional_source_tarballs' in gecko_config:
            extra_tarballs.extend(gecko_config['additional_source_tarballs'])

        for tarball in extra_tarballs:
            self.run_command(tar + ["xf", tarball], cwd=dirs['work_dir'],
                             halt_on_failure=True)

    def checkout_gaia(self):
        dirs = self.query_abs_dirs()
        gecko_config = self.load_gecko_config()
        gaia_config = gecko_config.get('gaia')
        if gaia_config:
            dest = os.path.join(dirs['abs_work_dir'], 'gaia')
            repo = gaia_config['repo']
            branch = gaia_config.get('branch')
            vcs = gaia_config['vcs']
            rev = self.vcs_checkout(repo=repo, dest=dest, vcs=vcs, branch=branch)
            self.set_buildbot_property('gaia_revision', rev, write_to_file=True)
            self.info("TinderboxPrint: gaia_revlink: %s/rev/%s" % (repo, rev))

    def checkout_gaia_l10n(self):
        if not self.config.get('gaia_languages_file'):
            self.info('Skipping checkout_gaia_l10n because no gaia language file was specified.')
            return

        l10n_config = self.load_gecko_config().get('gaia', {}).get('l10n')
        if not l10n_config:
            self.fatal("gaia.l10n is required in the gecko config when --gaia-languages-file is specified.")

        abs_work_dir = self.query_abs_dirs()['abs_work_dir']
        languages_file = os.path.join(abs_work_dir, 'gaia', self.config['gaia_languages_file'])
        l10n_base_dir = self.query_abs_dirs()['gaia_l10n_base_dir']

        self.pull_gaia_locale_source(l10n_config, parse_config_file(languages_file).keys(), l10n_base_dir)

    def checkout_gecko_l10n(self):
        hg_l10n_base = self.load_gecko_config().get('gecko_l10n_root')
        self.pull_locale_source(hg_l10n_base=hg_l10n_base)
        gecko_locales = self.query_locales()
        # populate b2g/overrides, which isn't in gecko atm
        dirs = self.query_abs_dirs()
        for locale in gecko_locales:
            self.mkdir_p(os.path.join(dirs['abs_l10n_dir'], locale, 'b2g', 'chrome', 'overrides'))
            self.copytree(os.path.join(dirs['abs_l10n_dir'], locale, 'mobile', 'overrides'),
                          os.path.join(dirs['abs_l10n_dir'], locale, 'b2g', 'chrome', 'overrides'),
                          error_level=FATAL)

    def checkout_compare_locales(self):
        dirs = self.query_abs_dirs()
        dest = dirs['compare_locales_dir']
        repo = self.config['compare_locales_repo']
        rev = self.config['compare_locales_rev']
        vcs = self.config['compare_locales_vcs']
        abs_rev = self.vcs_checkout(repo=repo, dest=dest, revision=rev, vcs=vcs)
        self.set_buildbot_property('compare_locales_revision', abs_rev, write_to_file=True)

    def query_do_translate_hg_to_git(self, gecko_config_key=None):
        manifest_config = self.config.get('manifest', {})
        branch = self.query_branch()
        if self.query_is_nightly() and branch in manifest_config['branches'] and \
                manifest_config.get('translate_hg_to_git'):
            if gecko_config_key is None:
                return True
            if self.gecko_config.get(gecko_config_key):
                return True
        return False

    def _generate_git_locale_manifest(self, locale, url, git_repo,
                                      revision, git_base_url, local_path):
        l10n_git_sha = self.query_translated_revision(url, 'l10n', revision, project_name="l10n %s" % locale)
        return '  <project name="%s" path="%s" remote="mozillaorg" revision="%s"/>' % (git_repo.replace(git_base_url, ''), local_path, l10n_git_sha)

    def _generate_locale_manifest(self, git_base_url="https://git.mozilla.org/release/"):
        """ Add the locales to the source manifest.
        """
        manifest_config = self.config.get('manifest', {})
        locale_manifest = []
        if self.gaia_locale_revisions:
            gaia_l10n_git_root = None
            if self.query_do_translate_hg_to_git(gecko_config_key='gaia_l10n_git_root'):
                gaia_l10n_git_root = self.gecko_config['gaia_l10n_git_root']
            for locale in self.gaia_locale_revisions.keys():
                repo = self.gaia_locale_revisions[locale]['repo']
                revision = self.gaia_locale_revisions[locale]['revision']
                locale_manifest.append('  <!-- Mercurial-Information: <project name="%s" path="gaia-l10n/%s" remote="hgmozillaorg" revision="%s"/> -->' %
                                       (repo.replace('http://hg.mozilla.org/', ''), locale, revision))
                if gaia_l10n_git_root:
                    locale_manifest.append(
                        self._generate_git_locale_manifest(
                            locale,
                            manifest_config['translate_base_url'],
                            gaia_l10n_git_root % {'locale': locale},
                            revision,
                            git_base_url,
                            "gaia-l10n/%s" % locale,
                        )
                    )
        if self.gecko_locale_revisions:
            gecko_l10n_git_root = None
            if self.query_do_translate_hg_to_git(gecko_config_key='gecko_l10n_git_root'):
                gecko_l10n_git_root = self.gecko_config['gecko_l10n_git_root']
            for locale in self.gecko_locale_revisions.keys():
                repo = self.gecko_locale_revisions[locale]['repo']
                revision = self.gecko_locale_revisions[locale]['revision']
                locale_manifest.append('  <!-- Mercurial-Information: <project name="%s" path="gecko-l10n/%s" remote="hgmozillaorg" revision="%s"/> -->' %
                                       (repo.replace('http://hg.mozilla.org/', ''), locale, revision))
                if gecko_l10n_git_root:
                    locale_manifest.append(
                        self._generate_git_locale_manifest(
                            locale,
                            manifest_config['translate_base_url'],
                            gecko_l10n_git_root % {'locale': locale},
                            revision,
                            git_base_url,
                            "gecko-l10n/%s" % locale,
                        )
                    )
        return locale_manifest

    def update_source_manifest(self):
        dirs = self.query_abs_dirs()
        gecko_config = self.load_gecko_config()
        gaia_config = gecko_config.get('gaia')
        manifest_config = self.config.get('manifest', {})

        sourcesfile = os.path.join(dirs['work_dir'], 'sources.xml')
        sourcesfile_orig = sourcesfile + '.original'
        sources = self.read_from_file(sourcesfile_orig, verbose=False)
        dom = xml.dom.minidom.parseString(sources)
        git_base_url = "https://git.mozilla.org/"
        for element in dom.getElementsByTagName('remote'):
            if element.getAttribute('name') == 'mozillaorg':
                pieces = urlparse.urlparse(element.getAttribute('fetch'))
                if pieces:
                    git_base_url = "https://git.mozilla.org%s" % pieces[2]
                    if not git_base_url.endswith('/'):
                        git_base_url += "/"
                    self.info("Found git_base_url of %s in manifest." % git_base_url)
                    break
        else:
            self.warning("Couldn't find git_base_url in manifest; using %s" % git_base_url)
        new_sources = []

        for line in sources.splitlines():
            new_sources.append(line)
            if 'Gonk specific things' in line:
                new_sources.append('  <!-- Mercurial-Information: <remote fetch="http://hg.mozilla.org/" name="hgmozillaorg"> -->')
                new_sources.append('  <!-- Mercurial-Information: <project name="%s" path="gecko" remote="hgmozillaorg" revision="%s"/> -->' %
                                   (self.buildbot_config['properties']['repo_path'], self.buildbot_properties['gecko_revision']))
                if gecko_config.get('config_version', 0) < 2 and 'gaia_revision' in self.buildbot_properties:
                    new_sources.append('  <!-- Mercurial-Information: <project name="%s" path="gaia" remote="hgmozillaorg" revision="%s"/> -->' %
                                       (gaia_config['repo'].replace('http://hg.mozilla.org/', ''), self.buildbot_properties['gaia_revision']))

                if self.query_do_translate_hg_to_git():
                    url = manifest_config['translate_base_url']
                    gecko_git = self.query_translated_revision(url, 'gecko', self.buildbot_properties['gecko_revision'])
                    new_sources.append('  <project name="%s" path="gecko" remote="mozillaorg" revision="%s"/>' % ("https://git.mozilla.org/releases/gecko.git".replace(git_base_url, ''), gecko_git))
                    # We don't need to use mapper for gaia after config version
                    # 2, since that's when we switched to pulling gaia directly
                    # from git
                    if gecko_config.get('config_version', 0) < 2:
                        if 'gaia_revision' in self.buildbot_properties:
                            gaia_git = self.query_translated_revision(url, 'gaia', self.buildbot_properties['gaia_revision'])
                            new_sources.append('  <project name="%s" path="gaia" remote="mozillaorg" revision="%s"/>' %
                                               ("https://git.mozilla.org/releases/gaia.git".replace(git_base_url, ''), gaia_git))

                        # Figure out when our gaia commit happened
                        gaia_time = self.get_hg_commit_time(os.path.join(dirs['abs_work_dir'], 'gaia'), self.buildbot_properties['gaia_revision'])

                        # Write the gaia commit information to 'gaia_commit_override.txt'
                        gaia_override = os.path.join(dirs['abs_work_dir'], 'gaia', 'gaia_commit_override.txt')
                        self.write_to_file(gaia_override, "%s\n%s\n" % (gaia_git, gaia_time))
                new_sources.extend(self._generate_locale_manifest(git_base_url=git_base_url))

        self.write_to_file(sourcesfile, "\n".join(new_sources), verbose=False)
        self.run_command(["diff", "-u", sourcesfile_orig, sourcesfile], success_codes=[1])

    def build(self):
        dirs = self.query_abs_dirs()
        gecko_config = self.load_gecko_config()
        build_targets = gecko_config.get('build_targets', [])
        cmd = ['./build.sh'] + build_targets
        env = self.query_build_env()
        if self.config.get('gaia_languages_file'):
            env['LOCALE_BASEDIR'] = dirs['gaia_l10n_base_dir']
            env['LOCALES_FILE'] = os.path.join(dirs['abs_work_dir'], 'gaia', self.config['gaia_languages_file'])
{'fraction_non_alphanumeric': 0.11822467110667391, 'fraction_numerical': 0.004739096917691365, 'mean_word_length': 1.9367229541658935, 'pattern_counts': {'":': 62, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 16, 'lorem ipsum': 0, 'www.': 0, 'xml': 52}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6846745', 'n_tokens_mistral': 24411, 'n_tokens_neox': 23076, 'n_words': 4554}