<commit_before>#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""multi_locale_build.py

This should be a mostly generic multilocale build script.
"""

from copy import deepcopy
import os
import sys

sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))

from mozharness.base.errors import MakefileErrorList, SSHErrorList
from mozharness.base.log import FATAL
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.l10n.locales import LocalesMixin



# MultiLocaleBuild {{{1
class MultiLocaleBuild(LocalesMixin, MercurialScript):
    """ This class targets Fennec multilocale builds.
        We were considering this for potential Firefox desktop multilocale.
        Now that we have a different approach for B2G multilocale,
        it's most likely misnamed. """
    config_options = [[
     ["--locale",],
     {"action": "extend",
      "dest": "locales",
      "type": "string",
      "help": "Specify the locale(s) to repack"
     }
    ],[
     ["--merge-locales",],
     {"action": "store_true",
      "dest": "merge_locales",
      "default": False,
      "help": "Use default [en-US] if there are missing strings"
     }
    ],[
     ["--no-merge-locales",],
     {"action": "store_false",
      "dest": "merge_locales",
      "help": "Do not allow missing strings"
     }
    ],[
     ["--objdir",],
     {"action": "store",
      "dest": "objdir",
      "type": "string",
      "default": "objdir",
      "help": "Specify the objdir"
     }
    ],[
     ["--l10n-base",],
     {"action": "store",
      "dest": "hg_l10n_base",
      "type": "string",
      "help": "Specify the L10n repo base directory"
     }
    ],[
     ["--l10n-tag",],
     {"action": "store",
      "dest": "hg_l10n_tag",
      "type": "string",
      "help": "Specify the L10n tag"
     }
    ],[
     ["--tag-override",],
     {"action": "store",
      "dest": "tag_override",
      "type": "string",
      "help": "Override the tags set for all repos"
     }
    ],[
     ["--user-repo-override",],
     {"action": "store",
      "dest": "user_repo_override",
      "type": "string",
      "help": "Override the user repo path for all repos"
     }
    ],[
     ["--l10n-dir",],
     {"action": "store",
      "dest": "l10n_dir",
      "type": "string",
      "default": "l10n",
      "help": "Specify the l10n dir name"
     }
    ]]

    def __init__(self, require_config_file=True):
        LocalesMixin.__init__(self)
        MercurialScript.__init__(self, config_options=self.config_options,
                                 all_actions=['clobber', 'pull-build-source',
                                              'pull-locale-source',
                                              'build', 'package-en-US',
                                              'upload-en-US',
                                              'backup-objdir',
                                              'restore-objdir',
                                              'add-locales', 'package-multi',
                                              'upload-multi', 'summary'],
                                 require_config_file=require_config_file)

    def clobber(self):
        c = self.config
        if c['work_dir'] != '.':
            path = os.path.join(c['base_work_dir'], c['work_dir'])
            if os.path.exists(path):
                self.rmtree(path, error_level=FATAL)
        else:
            self.info("work_dir is '.'; skipping for now.")

    def pull_build_source(self):
        c = self.config
        repos = []
        replace_dict = {}
        # Replace %(user_repo_override)s with c['user_repo_override']
        if c.get("user_repo_override"):
            replace_dict['user_repo_override'] = c['user_repo_override']
            for repo_dict in deepcopy(c['repos']):
                repo_dict['repo'] = repo_dict['repo'] % replace_dict
                repos.append(repo_dict)
        else:
            repos = c['repos']
        self.vcs_checkout_repos(repos, tag_override=c.get('tag_override'))

    # pull_locale_source() defined in LocalesMixin.

    def build(self):
        c = self.config
        dirs = self.query_abs_dirs()
        self.copyfile(os.path.join(dirs['abs_work_dir'], c['mozconfig']),
                      os.path.join(dirs['abs_mozilla_dir'], 'mozconfig'),
                      error_level=FATAL)
        command = "make -f client.mk build"
        env = self.query_env()
        if self._process_command(command=command,
                                 cwd=dirs['abs_mozilla_dir'],
                                 env=env, error_list=MakefileErrorList):
            self.fatal("Erroring out after the build failed.")

    def add_locales(self):
        c = self.config
        dirs = self.query_abs_dirs()
        locales = self.query_locales()

        for locale in locales:
            self.run_compare_locales(locale, halt_on_failure=True)
            command = 'make chrome-%s L10NBASEDIR=%s' % (locale, dirs['abs_l10n_dir'])
            if c['merge_locales']:
                command += " LOCALE_MERGEDIR=%s" % dirs['abs_merge_dir']
            status = self._process_command(command=command,
                                           cwd=dirs['abs_locales_dir'],
                                           error_list=MakefileErrorList)
            if status:
                self.return_code += 1
                self.add_summary("Failed to add locale %s!" % locale,
                                 level="error")
            else:
                self.add_summary("Added locale %s successfully." % locale)

    def package_en_US(self):
        self.package(package_type='en-US')

    def preflight_package_multi(self):
        dirs = self.query_abs_dirs()
        self.run_command("rm -rfv dist/fennec*", cwd=dirs['abs_objdir'])

    def package_multi(self):
        self.package(package_type='multi')
<commit_msg>bug 933290 - fix multilocale android builds. r=callek r=glandium<commit_after>#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""multi_locale_build.py

This should be a mostly generic multilocale build script.
"""

from copy import deepcopy
import os
import sys

sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))

from mozharness.base.errors import MakefileErrorList, SSHErrorList
from mozharness.base.log import FATAL
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.l10n.locales import LocalesMixin



# MultiLocaleBuild {{{1
class MultiLocaleBuild(LocalesMixin, MercurialScript):
    """ This class targets Fennec multilocale builds.
        We were considering this for potential Firefox desktop multilocale.
        Now that we have a different approach for B2G multilocale,
        it's most likely misnamed. """
    config_options = [[
     ["--locale",],
     {"action": "extend",
      "dest": "locales",
      "type": "string",
      "help": "Specify the locale(s) to repack"
     }
    ],[
     ["--merge-locales",],
     {"action": "store_true",
      "dest": "merge_locales",
      "default": False,
      "help": "Use default [en-US] if there are missing strings"
     }
    ],[
     ["--no-merge-locales",],
     {"action": "store_false",
      "dest": "merge_locales",
      "help": "Do not allow missing strings"
     }
    ],[
     ["--objdir",],
     {"action": "store",
      "dest": "objdir",
      "type": "string",
      "default": "objdir",
      "help": "Specify the objdir"
     }
    ],[
     ["--l10n-base",],
     {"action": "store",
      "dest": "hg_l10n_base",
      "type": "string",
      "help": "Specify the L10n repo base directory"
     }
    ],[
     ["--l10n-tag",],
     {"action": "store",
      "dest": "hg_l10n_tag",
      "type": "string",
      "help": "Specify the L10n tag"
     }
    ],[
     ["--tag-override",],
     {"action": "store",
      "dest": "tag_override",
      "type": "string",
      "help": "Override the tags set for all repos"
     }
    ],[
     ["--user-repo-override",],
     {"action": "store",
      "dest": "user_repo_override",
      "type": "string",
      "help": "Override the user repo path for all repos"
     }
    ],[
     ["--l10n-dir",],
     {"action": "store",
      "dest": "l10n_dir",
      "type": "string",
      "default": "l10n",
      "help": "Specify the l10n dir name"
     }
    ]]

    def __init__(self, require_config_file=True):
        LocalesMixin.__init__(self)
        MercurialScript.__init__(self, config_options=self.config_options,
                                 all_actions=['clobber', 'pull-build-source',
                                              'pull-locale-source',
                                              'build', 'package-en-US',
                                              'upload-en-US',
                                              'backup-objdir',
                                              'restore-objdir',
                                              'add-locales', 'package-multi',
                                              'upload-multi', 'summary'],
                                 require_config_file=require_config_file)

    def clobber(self):
        c = self.config
        if c['work_dir'] != '.':
            path = os.path.join(c['base_work_dir'], c['work_dir'])
            if os.path.exists(path):
                self.rmtree(path, error_level=FATAL)
        else:
            self.info("work_dir is '.'; skipping for now.")

    def pull_build_source(self):
        c = self.config
        repos = []
        replace_dict = {}
        # Replace %(user_repo_override)s with c['user_repo_override']
        if c.get("user_repo_override"):
            replace_dict['user_repo_override'] = c['user_repo_override']
            for repo_dict in deepcopy(c['repos']):
                repo_dict['repo'] = repo_dict['repo'] % replace_dict
                repos.append(repo_dict)
        else:
            repos = c['repos']
        self.vcs_checkout_repos(repos, tag_override=c.get('tag_override'))

    # pull_locale_source() defined in LocalesMixin.

    def build(self):
        c = self.config
        dirs = self.query_abs_dirs()
        self.copyfile(os.path.join(dirs['abs_work_dir'], c['mozconfig']),
                      os.path.join(dirs['abs_mozilla_dir'], 'mozconfig'),
                      error_level=FATAL)
        command = "make -f client.mk build"
        env = self.query_env()
        if self._process_command(command=command,
                                 cwd=dirs['abs_mozilla_dir'],
                                 env=env, error_list=MakefileErrorList):
            self.fatal("Erroring out after the build failed.")

    def add_locales(self):
        c = self.config
        dirs = self.query_abs_dirs()
        locales = self.query_locales()

        for locale in locales:
            self.run_compare_locales(locale, halt_on_failure=True)
            command = 'make chrome-%s L10NBASEDIR=%s' % (locale, dirs['abs_l10n_dir'])
            if c['merge_locales']:
                command += " LOCALE_MERGEDIR=%s" % dirs['abs_merge_dir']
            status = self._process_command(command=command,
                                           cwd=dirs['abs_locales_dir'],
                                           error_list=MakefileErrorList)
            if status:
                self.return_code += 1
                self.add_summary("Failed to add locale %s!" % locale,
                                 level="error")
            else:
                self.add_summary("Added locale %s successfully." % locale)

    def package_en_US(self):
        self.package(package_type='en-US')

    def preflight_package_multi(self):
        dirs = self.query_abs_dirs()
        self.run_command("rm -rfv dist/fennec*", cwd=dirs['abs_objdir'])
        # bug 933290
        self.run_command(["touch", "mobile/android/installer/Makefile"], cwd=dirs['abs_objdir'])

    def package_multi(self):
        self.package(package_type='multi')

{'fraction_non_alphanumeric': 0.13617706237424548, 'fraction_numerical': 0.006599597585513079, 'mean_word_length': 1.9244528124264533, 'pattern_counts': {'":': 74, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3304592', 'n_tokens_mistral': 3823, 'n_tokens_neox': 3661, 'n_words': 806}