<commit_before>#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****

import sys
import os
import glob
import re
import tempfile
from datetime import datetime
import urlparse
import multiprocessing
import xml.dom.minidom

try:
    import simplejson as json
    assert json
except ImportError:
    import json

# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))

# import the guts
from mozharness.base.config import parse_config_file
from mozharness.base.log import WARNING, ERROR, FATAL
from mozharness.mozilla.l10n.locales import GaiaLocalesMixin, LocalesMixin
from mozharness.mozilla.purge import PurgeMixin
from mozharness.mozilla.signing import SigningMixin
from mozharness.mozilla.repo_manifest import add_project
from mozharness.mozilla.mapper import MapperMixin
from mozharness.mozilla.updates.balrog import BalrogMixin
from mozharness.base.python import VirtualenvMixin
from mozharness.base.python import InfluxRecordingMixin
from mozharness.mozilla.building.buildbase import MakeUploadOutputParser
from mozharness.mozilla.building.buildb2gbase import B2GBuildBaseScript, B2GMakefileErrorList


class B2GBuild(LocalesMixin, PurgeMixin,
               B2GBuildBaseScript,
               GaiaLocalesMixin, SigningMixin, MapperMixin, BalrogMixin,
               VirtualenvMixin, InfluxRecordingMixin):
    all_actions = [
        'clobber',
        'checkout-sources',
        # Deprecated
        'checkout-gecko',
        'download-gonk',
        'unpack-gonk',
        'checkout-gaia',
        'checkout-gaia-l10n',
        'checkout-gecko-l10n',
        'checkout-compare-locales',
        # End deprecated
        'get-blobs',
        'update-source-manifest',
        'build',
        'build-symbols',
        'make-updates',
        'build-update-testdata',
        'prep-upload',
        'upload',
        'make-socorro-json',
        'upload-source-manifest',
        'submit-to-balrog',
    ]

    default_actions = [
        'checkout-sources',
        'get-blobs',
        'build',
    ]

    config_options = [
        [["--gaia-languages-file"], {
            "dest": "gaia_languages_file",
            "help": "languages file for gaia multilocale profile",
        }],
        [["--gecko-languages-file"], {
            "dest": "locales_file",
            "help": "languages file for gecko multilocale",
        }],
        [["--gecko-l10n-base-dir"], {
            "dest": "l10n_dir",
            "help": "dir to clone gecko l10n repos into, relative to the work directory",
        }],
        [["--merge-locales"], {
            "dest": "merge_locales",
            "help": "Dummy option to keep from burning. We now always merge",
        }],
        [["--additional-source-tarballs"], {
            "action": "extend",
            "type": "string",
            "dest": "additional_source_tarballs",
            "help": "Additional source tarballs to extract",
        }],
        [["--debug"], {
            "dest": "debug_build",
            "action": "store_true",
            "help": "Set B2G_DEBUG=1 (debug build)",
        }],
        [["--base-repo"], {
            "dest": "base_repo",
            "help": "base repository for cloning",
        }],
        [["--complete-mar-url"], {
            "dest": "complete_mar_url",
            "help": "the URL where the complete MAR was uploaded. Required if submit-to-balrog is requested and upload isn't.",
        }],
        [["--platform"], {
            "dest": "platform",
            "help": "the platform used by balrog submmiter.",
        }],
    ]

    def __init__(self, require_config_file=False, config={},
                 all_actions=all_actions,
                 default_actions=default_actions):
        # Default configuration
        default_config = {
            'default_vcs': 'hgtool',
            'ccache': True,
            'locales_dir': 'gecko/b2g/locales',
            'l10n_dir': 'gecko-l10n',
            'ignore_locales': ['en-US', 'multi'],
            'locales_file': 'gecko/b2g/locales/all-locales',
            'mozilla_dir': 'build/gecko',
            'objdir': 'build/objdir-gecko',
            'merge_locales': True,
            'compare_locales_repo': 'https://hg.mozilla.org/build/compare-locales',
            'compare_locales_rev': 'RELEASE_AUTOMATION',
            'compare_locales_vcs': 'hgtool',
            'repo_remote_mappings': {},
            'influx_credentials_file': 'oauth.txt',
            'balrog_credentials_file': 'oauth.txt',
            'virtualenv_modules': [
                'requests==2.2.1',
            ],
            'virtualenv_path': 'venv',
        }
        default_config.update(config)

        self.buildid = None
        self.dotconfig = None
        LocalesMixin.__init__(self)
        B2GBuildBaseScript.__init__(
            self,
            config_options=self.config_options,
            require_config_file=require_config_file,
            config=default_config,
            all_actions=all_actions,
            default_actions=default_actions,
        )

        dirs = self.query_abs_dirs()
        self.objdir = os.path.join(dirs['work_dir'], 'objdir-gecko')
        if self.config.get("update_type", "ota") == "fota":
            self.make_updates_cmd = ['./build.sh', 'gecko-update-fota']
            self.extra_update_attrs = 'isOsUpdate="true"'
            self.isOSUpdate = True
        else:
            self.make_updates_cmd = ['./build.sh', 'gecko-update-full']
            self.extra_update_attrs = None
            self.isOSUpdate = False
        self.package_urls = {}

        # We need to create the virtualenv directly (without using an action) in
        # order to use python modules in PreScriptRun/Action listeners
        self.create_virtualenv()

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = LocalesMixin.query_abs_dirs(self)
        abs_dirs.update(B2GBuildBaseScript.query_abs_dirs(self))

        dirs = {
            'gaia_l10n_base_dir': os.path.join(abs_dirs['abs_work_dir'], 'gaia-l10n'),
            'compare_locales_dir': os.path.join(abs_dirs['abs_work_dir'], 'compare-locales'),
            'abs_public_upload_dir': os.path.join(abs_dirs['abs_work_dir'], 'upload-public'),
        }

        abs_dirs.update(dirs)
        self.abs_dirs = abs_dirs
        return self.abs_dirs

    def query_branch(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['branch']
        else:
            return os.path.basename(self.query_repo())

    def query_buildid(self):
        if self.buildid:
            return self.buildid<commit_msg>Bug 1156278: Make query_branch() more robust.<commit_after>#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****

import sys
import os
import glob
import re
import tempfile
from datetime import datetime
import urlparse
import multiprocessing
import xml.dom.minidom

try:
    import simplejson as json
    assert json
except ImportError:
    import json

# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))

# import the guts
from mozharness.base.config import parse_config_file
from mozharness.base.log import WARNING, ERROR, FATAL
from mozharness.mozilla.l10n.locales import GaiaLocalesMixin, LocalesMixin
from mozharness.mozilla.purge import PurgeMixin
from mozharness.mozilla.signing import SigningMixin
from mozharness.mozilla.repo_manifest import add_project
from mozharness.mozilla.mapper import MapperMixin
from mozharness.mozilla.updates.balrog import BalrogMixin
from mozharness.base.python import VirtualenvMixin
from mozharness.base.python import InfluxRecordingMixin
from mozharness.mozilla.building.buildbase import MakeUploadOutputParser
from mozharness.mozilla.building.buildb2gbase import B2GBuildBaseScript, B2GMakefileErrorList


class B2GBuild(LocalesMixin, PurgeMixin,
               B2GBuildBaseScript,
               GaiaLocalesMixin, SigningMixin, MapperMixin, BalrogMixin,
               VirtualenvMixin, InfluxRecordingMixin):
    all_actions = [
        'clobber',
        'checkout-sources',
        # Deprecated
        'checkout-gecko',
        'download-gonk',
        'unpack-gonk',
        'checkout-gaia',
        'checkout-gaia-l10n',
        'checkout-gecko-l10n',
        'checkout-compare-locales',
        # End deprecated
        'get-blobs',
        'update-source-manifest',
        'build',
        'build-symbols',
        'make-updates',
        'build-update-testdata',
        'prep-upload',
        'upload',
        'make-socorro-json',
        'upload-source-manifest',
        'submit-to-balrog',
    ]

    default_actions = [
        'checkout-sources',
        'get-blobs',
        'build',
    ]

    config_options = [
        [["--gaia-languages-file"], {
            "dest": "gaia_languages_file",
            "help": "languages file for gaia multilocale profile",
        }],
        [["--gecko-languages-file"], {
            "dest": "locales_file",
            "help": "languages file for gecko multilocale",
        }],
        [["--gecko-l10n-base-dir"], {
            "dest": "l10n_dir",
            "help": "dir to clone gecko l10n repos into, relative to the work directory",
        }],
        [["--merge-locales"], {
            "dest": "merge_locales",
            "help": "Dummy option to keep from burning. We now always merge",
        }],
        [["--additional-source-tarballs"], {
            "action": "extend",
            "type": "string",
            "dest": "additional_source_tarballs",
            "help": "Additional source tarballs to extract",
        }],
        [["--debug"], {
            "dest": "debug_build",
            "action": "store_true",
            "help": "Set B2G_DEBUG=1 (debug build)",
        }],
        [["--base-repo"], {
            "dest": "base_repo",
            "help": "base repository for cloning",
        }],
        [["--complete-mar-url"], {
            "dest": "complete_mar_url",
            "help": "the URL where the complete MAR was uploaded. Required if submit-to-balrog is requested and upload isn't.",
        }],
        [["--platform"], {
            "dest": "platform",
            "help": "the platform used by balrog submmiter.",
        }],
    ]

    def __init__(self, require_config_file=False, config={},
                 all_actions=all_actions,
                 default_actions=default_actions):
        # Default configuration
        default_config = {
            'default_vcs': 'hgtool',
            'ccache': True,
            'locales_dir': 'gecko/b2g/locales',
            'l10n_dir': 'gecko-l10n',
            'ignore_locales': ['en-US', 'multi'],
            'locales_file': 'gecko/b2g/locales/all-locales',
            'mozilla_dir': 'build/gecko',
            'objdir': 'build/objdir-gecko',
            'merge_locales': True,
            'compare_locales_repo': 'https://hg.mozilla.org/build/compare-locales',
            'compare_locales_rev': 'RELEASE_AUTOMATION',
            'compare_locales_vcs': 'hgtool',
            'repo_remote_mappings': {},
            'influx_credentials_file': 'oauth.txt',
            'balrog_credentials_file': 'oauth.txt',
            'virtualenv_modules': [
                'requests==2.2.1',
            ],
            'virtualenv_path': 'venv',
        }
        default_config.update(config)

        self.buildid = None
        self.dotconfig = None
        LocalesMixin.__init__(self)
        B2GBuildBaseScript.__init__(
            self,
            config_options=self.config_options,
            require_config_file=require_config_file,
            config=default_config,
            all_actions=all_actions,
            default_actions=default_actions,
        )

        dirs = self.query_abs_dirs()
        self.objdir = os.path.join(dirs['work_dir'], 'objdir-gecko')
        if self.config.get("update_type", "ota") == "fota":
            self.make_updates_cmd = ['./build.sh', 'gecko-update-fota']
            self.extra_update_attrs = 'isOsUpdate="true"'
            self.isOSUpdate = True
        else:
            self.make_updates_cmd = ['./build.sh', 'gecko-update-full']
            self.extra_update_attrs = None
            self.isOSUpdate = False
        self.package_urls = {}

        # We need to create the virtualenv directly (without using an action) in
        # order to use python modules in PreScriptRun/Action listeners
        self.create_virtualenv()

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = LocalesMixin.query_abs_dirs(self)
        abs_dirs.update(B2GBuildBaseScript.query_abs_dirs(self))

        dirs = {
            'gaia_l10n_base_dir': os.path.join(abs_dirs['abs_work_dir'], 'gaia-l10n'),
            'compare_locales_dir': os.path.join(abs_dirs['abs_work_dir'], 'compare-locales'),
            'abs_public_upload_dir': os.path.join(abs_dirs['abs_work_dir'], 'upload-public'),
        }

        abs_dirs.update(dirs)
        self.abs_dirs = abs_dirs
        return self.abs_dirs

    def query_branch(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['branch']
        else:
            return os.path.basename(self.query_repo().rstrip('/'))

    def query_buildid(self):
        if self.buildid:
            return self.buildid
{'fraction_non_alphanumeric': 0.12102460785724564, 'fraction_numerical': 0.006259893509857534, 'mean_word_length': 2.277293091252063, 'pattern_counts': {'":': 44, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6397188', 'n_tokens_mistral': 4402, 'n_tokens_neox': 4263, 'n_words': 862}