<commit_before>#! /usr/bin/env python

#
# update-xliff.py <base_l10n_folder>
#
#  For each folder (locale) available in base_l10n_folder:
#
#  1. Read existing translations.
#
#  2. Inject available translations in the reference XLIFF file, updating
#     the target-language where available on file elements.
#
#  3. Store the updated content in existing locale files, without backup.
#

from glob import glob
from lxml import etree
import argparse
import os

NS = {'x':'urn:oasis:names:tc:xliff:document:1.2'}

def indent(elem, level=0):
    # Prettify XML output
    # http://effbot.org/zone/element-lib.htm#prettyprint
    i = '\n' + level*'  '
    if len(elem):
        if not elem.text or not elem.text.strip():
            elem.text = i + '  '
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
        for elem in elem:
            indent(elem, level+1)
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
    else:
        if level and (not elem.tail or not elem.tail.strip()):
            elem.tail = i

def main():
    # Base parameters, there should be no need to change these unless
    # there are more locales to exclude.
    reference_locale = 'en-US'
    target_language = 'en'
    xliff_filename = 'firefox-ios.xliff'
    excluded_locales = ['pl', reference_locale]

    parser = argparse.ArgumentParser()
    parser.add_argument('base_folder', help='Path to folder including subfolders for all locales')
    args = parser.parse_args()

    # Get a list of files to update (absolute paths)
    base_folder = os.path.realpath(args.base_folder)
    file_paths = []
    for xliff_path in glob(base_folder + '/*/' + xliff_filename):
        parts = xliff_path.split(os.sep)
        if not parts[-2] in excluded_locales:
            file_paths.append(xliff_path)
    file_paths.sort()

    for file_path in file_paths:
        print 'Updating %s' % file_path

        # Read the reference file XML
        reference_tree = etree.parse(os.path.join(base_folder, reference_locale, xliff_filename))
        reference_root = reference_tree.getroot()

        # Read localized file XML
        locale_tree = etree.parse(file_path)
        locale_root = locale_tree.getroot()

        # Using locale folder as locale code. In some cases we need to map this
        # value to a different locale code
        # http://www.ibabbleon.com/iOS-Language-Codes-ISO-639.html
        locale_code = file_path.split(os.sep)[-2]
        locale_mapping = {
            'bn-IN': 'bn',
            'ga-IE': 'ga',
            'sv-SE': 'sv'
        }
        if locale_code in locale_mapping:
            locale_code = locale_mapping[locale_code]

        # Store existing localizations
        translations = {}
        for trans_node in locale_root.xpath('//x:trans-unit', namespaces=NS):
            file_name = trans_node.getparent().getparent().get('original')
            string_id = '%s:%s' % (file_name, trans_node.get('id'))
            for child in trans_node.xpath('./x:target', namespaces=NS):
                translations[string_id] = child.text

        # Inject available translations in the reference XML
        for trans_node in reference_root.xpath('//x:trans-unit', namespaces=NS):
            file_name = trans_node.getparent().getparent().get('original')
            string_id = '%s:%s' % (file_name, trans_node.get('id'))

            updated = False
            translated = string_id in translations
            for child in trans_node.xpath('./x:target', namespaces=NS):
                if translated:
                    # Translation is available, update the target
                    child.text = translations[string_id]
                else:
                    # No translation available, remove the target
                    child.getparent().remove(child)
                updated = True

            if translated and not updated:
                # Translation is available, but reference has no target.
                # Create a target node and insert it after source.<commit_msg>Bug 1192909 - [l10n] Update script: discard translation if source string has changed<commit_after>#! /usr/bin/env python

#
# update-xliff.py <base_l10n_folder>
#
#  For each folder (locale) available in base_l10n_folder:
#
#  1. Read existing translations, store them in an array: IDs use the structure
#     file_name:string_id:source_hash. Using the hash of the source string
#     prevent from keeping an existing translation if the ID doesn't change
#     but the source string does.
#
#  2. Inject available translations in the reference XLIFF file, updating
#     the target-language where available on file elements.
#
#  3. Store the updated content in existing locale files, without backup.
#

from glob import glob
from lxml import etree
import argparse
import os

NS = {'x':'urn:oasis:names:tc:xliff:document:1.2'}

def indent(elem, level=0):
    # Prettify XML output
    # http://effbot.org/zone/element-lib.htm#prettyprint
    i = '\n' + level*'  '
    if len(elem):
        if not elem.text or not elem.text.strip():
            elem.text = i + '  '
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
        for elem in elem:
            indent(elem, level+1)
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
    else:
        if level and (not elem.tail or not elem.tail.strip()):
            elem.tail = i

def main():
    # Base parameters, there should be no need to change these unless
    # there are more locales to exclude.
    reference_locale = 'en-US'
    target_language = 'en'
    xliff_filename = 'firefox-ios.xliff'
    excluded_locales = ['pl', reference_locale]

    parser = argparse.ArgumentParser()
    parser.add_argument('base_folder', help='Path to folder including subfolders for all locales')
    args = parser.parse_args()

    # Get a list of files to update (absolute paths)
    base_folder = os.path.realpath(args.base_folder)
    file_paths = []
    for xliff_path in glob(base_folder + '/*/' + xliff_filename):
        parts = xliff_path.split(os.sep)
        if not parts[-2] in excluded_locales:
            file_paths.append(xliff_path)
    file_paths.sort()

    for file_path in file_paths:
        print 'Updating %s' % file_path

        # Read the reference file XML
        reference_tree = etree.parse(os.path.join(base_folder, reference_locale, xliff_filename))
        reference_root = reference_tree.getroot()

        # Read localized file XML
        locale_tree = etree.parse(file_path)
        locale_root = locale_tree.getroot()

        # Using locale folder as locale code. In some cases we need to map this
        # value to a different locale code
        # http://www.ibabbleon.com/iOS-Language-Codes-ISO-639.html
        locale_code = file_path.split(os.sep)[-2]
        locale_mapping = {
            'bn-IN': 'bn',
            'ga-IE': 'ga',
            'sv-SE': 'sv'
        }
        if locale_code in locale_mapping:
            locale_code = locale_mapping[locale_code]

        # Store existing localizations
        translations = {}
        for trans_node in locale_root.xpath('//x:trans-unit', namespaces=NS):
            for child in trans_node.xpath('./x:target', namespaces=NS):
                file_name = trans_node.getparent().getparent().get('original')
                source_string = trans_node.xpath('./x:source', namespaces=NS)[0].text
                string_id = '%s:%s:%s' % (file_name, trans_node.get('id'), hash(source_string))
                translations[string_id] = child.text

        # Inject available translations in the reference XML
        for trans_node in reference_root.xpath('//x:trans-unit', namespaces=NS):
            file_name = trans_node.getparent().getparent().get('original')
            source_string = trans_node.xpath('./x:source', namespaces=NS)[0].text
            string_id = '%s:%s:%s' % (file_name, trans_node.get('id'), hash(source_string))
            updated = False
            translated = string_id in translations
            for child in trans_node.xpath('./x:target', namespaces=NS):
                if translated:
                    # Translation is available, update the target
                    child.text = translations[string_id]
                else:
                    # No translation available, remove the target
                    child.getparent().remove(child)
                updated = True

            if translated and not updated:
                # Translation is available, but reference has no target.
                # Create a target node and insert it after source.
{'fraction_non_alphanumeric': 0.10146939720004627, 'fraction_numerical': 0.004975124378109453, 'mean_word_length': 2.3555900621118013, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6779547', 'n_tokens_mistral': 2547, 'n_tokens_neox': 2464, 'n_words': 719}