<commit_before>#!/usr/bin/env python
#
#    Copyright 2012-2013 <NAME>
#
#    This file is part of OSPFM-web.
#
#    OSPFM-web is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    OSPFM-web is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with OSPFM-web.  If not, see <http://www.gnu.org/licenses/>.

###############################################################################

SOURCEDIR='src/js'
TRANSLATIONDIR='src/locale'

NUMBERS = (
    ('decimal', '.'),
    ('digits', 3),
    ('separator', ''),
    ('curdecimals', 2),
    ('posnum', 'N'),
    ('poscur', 'NC'),
    ('pospct', 'N%'),
    ('negnum', '-N'),
    ('negcur', '-NC'),
    ('negpct', '-N%'),
)

NAME = (
    ('nameandnick', 'F L (N)'),
    ('fullname', 'F L'),
    ('shortname', 'F'),
    ('nickname', '(N)'),
)

DATE = (
    ('first_day_of_week', 'monday'),
    ('shortformat', '%d/%m/%Y'),
)

CURRENCY = (
    ('completename', 'C: N (S)'),
)

CURRENCIES = ('AED', 'AFN', 'ALL', 'AMD', 'ANG', 'AOA', 'ARS', 'AUD', 'AWG', 'AZN', 'BAM', 'BBD', 'BDT', 'BGN', 'BHD', 'BIF', 'BMD', 'BND', 'BOB', 'BRL', 'BSD', 'BTN', 'BWP', 'BYR', 'BZD', 'CAD', 'CDF', 'CHF', 'CLF', 'CLP', 'CNY', 'COP', 'CRC', 'CUP', 'CVE', 'CZK', 'DJF', 'DKK', 'DOP', 'DZD', 'EGP', 'ETB', 'EUR', 'FJD', 'FKP', 'GBP', 'GEL', 'GHS', 'GIP', 'GMD', 'GNF', 'GTQ', 'GYD', 'HKD', 'HNL', 'HRK', 'HTG', 'HUF', 'IDR', 'ILS', 'INR', 'IQD', 'IRR', 'ISK', 'JMD', 'JOD', 'JPY', 'KES', 'KGS', 'KHR', 'KMF', 'KPW', 'KRW', 'KWD', 'KZT', 'LAK', 'LBP', 'LKR', 'LRD', 'LSL', 'LTL', 'LVL', 'LYD', 'MAD', 'MDL', 'MGA', 'MKD', 'MMK', 'MNT', 'MOP', 'MRO', 'MUR', 'MVR', 'MWK', 'MXN', 'MYR', 'MZN', 'NAD', 'NGN', 'NIO', 'NOK', 'NPR', 'NZD', 'OMR', 'PAB', 'PEN', 'PGK', 'PHP', 'PKR', 'PLN', 'PYG', 'QAR', 'RON', 'RSD', 'RUB', 'RWF', 'SAR', 'SBD', 'SCR', 'SDG', 'SEK', 'SGD', 'SHP', 'SLL', 'SOS', 'SRD', 'STD', 'SYP', 'SZL', 'THB', 'TJS', 'TMT', 'TND', 'TOP', 'TRY', 'TTD', 'TWD', 'TZS', 'UAH', 'UGX', 'USD', 'UYI', 'UYU', 'UZS', 'VEF', 'VND', 'VUV', 'WST', 'XAF')

###############################################################################

import codecs
import datetime
import json
import os
import re
import shutil
import tempfile



def get_all_strings():
    strings = {}
    for dirdata in os.walk(SOURCEDIR):
        for filename in dirdata[2]:
            if filename.endswith('.js'):
                for string in get_strings(
                                    os.path.join(dirdata[0], filename)
                                ).items():
                    if string[0] in strings:
                        strings[string[0]].extend(string[1])
                    else:
                        strings[string[0]] = string[1]
    return strings



def get_strings(filename):
    strings = {}
    with open(filename, 'r') as f:
        for linenb, line in enumerate(f.readlines()):
            for string in re.findall("_\('([^\']*)'\)", line):
                if string in strings:
                    strings[string].append('%s:%d' % (filename, linenb))
                else:
                    strings[string] = ['%s:%d' % (filename, linenb)]
    return strings



def update_all_translation_files(strings):
    locales = []
    files = []
    for filename in os.listdir(TRANSLATIONDIR):
        if filename.endswith('.js'):
            files.append(filename)
            locales.append(filename[:-3])
    targetdir = tempfile.mkdtemp()
    for filename in files:
        update_translation_file(
            filename,
            targetdir,
            strings,
            locales
        )
    shutil.rmtree(targetdir)



def entries(name, currentvalues, defaultvalues):
    if name in currentvalues:
        already_defined = currentvalues[name]<commit_msg>Generate translations from OSPFM API error messages<commit_after>#!/usr/bin/env python
#
#    Copyright 2012-2013 <NAME>
#
#    This file is part of OSPFM-web.
#
#    OSPFM-web is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    OSPFM-web is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with OSPFM-web.  If not, see <http://www.gnu.org/licenses/>.

###############################################################################

SOURCEDIR='src/js'
TRANSLATIONDIR='src/locale'
OSPFMSOURCEDIR='../ospfm/ospfm'

NUMBERS = (
    ('decimal', '.'),
    ('digits', 3),
    ('separator', ''),
    ('curdecimals', 2),
    ('posnum', 'N'),
    ('poscur', 'NC'),
    ('pospct', 'N%'),
    ('negnum', '-N'),
    ('negcur', '-NC'),
    ('negpct', '-N%'),
)

NAME = (
    ('nameandnick', 'F L (N)'),
    ('fullname', 'F L'),
    ('shortname', 'F'),
    ('nickname', '(N)'),
)

DATE = (
    ('first_day_of_week', 'monday'),
    ('shortformat', '%d/%m/%Y'),
)

CURRENCY = (
    ('completename', 'C: N (S)'),
)

CURRENCIES = ('AED', 'AFN', 'ALL', 'AMD', 'ANG', 'AOA', 'ARS', 'AUD', 'AWG', 'AZN', 'BAM', 'BBD', 'BDT', 'BGN', 'BHD', 'BIF', 'BMD', 'BND', 'BOB', 'BRL', 'BSD', 'BTN', 'BWP', 'BYR', 'BZD', 'CAD', 'CDF', 'CHF', 'CLF', 'CLP', 'CNY', 'COP', 'CRC', 'CUP', 'CVE', 'CZK', 'DJF', 'DKK', 'DOP', 'DZD', 'EGP', 'ETB', 'EUR', 'FJD', 'FKP', 'GBP', 'GEL', 'GHS', 'GIP', 'GMD', 'GNF', 'GTQ', 'GYD', 'HKD', 'HNL', 'HRK', 'HTG', 'HUF', 'IDR', 'ILS', 'INR', 'IQD', 'IRR', 'ISK', 'JMD', 'JOD', 'JPY', 'KES', 'KGS', 'KHR', 'KMF', 'KPW', 'KRW', 'KWD', 'KZT', 'LAK', 'LBP', 'LKR', 'LRD', 'LSL', 'LTL', 'LVL', 'LYD', 'MAD', 'MDL', 'MGA', 'MKD', 'MMK', 'MNT', 'MOP', 'MRO', 'MUR', 'MVR', 'MWK', 'MXN', 'MYR', 'MZN', 'NAD', 'NGN', 'NIO', 'NOK', 'NPR', 'NZD', 'OMR', 'PAB', 'PEN', 'PGK', 'PHP', 'PKR', 'PLN', 'PYG', 'QAR', 'RON', 'RSD', 'RUB', 'RWF', 'SAR', 'SBD', 'SCR', 'SDG', 'SEK', 'SGD', 'SHP', 'SLL', 'SOS', 'SRD', 'STD', 'SYP', 'SZL', 'THB', 'TJS', 'TMT', 'TND', 'TOP', 'TRY', 'TTD', 'TWD', 'TZS', 'UAH', 'UGX', 'USD', 'UYI', 'UYU', 'UZS', 'VEF', 'VND', 'VUV', 'WST', 'XAF')

###############################################################################

import codecs
import datetime
import json
import os
import re
import shutil
import tempfile



def get_all_strings():
    strings = {}
    for dirdata in os.walk(SOURCEDIR):
        for filename in dirdata[2]:
            if filename.endswith('.js'):
                for string in get_strings(
                                    os.path.join(dirdata[0], filename)
                                ).items():
                    if string[0] in strings:
                        strings[string[0]].extend(string[1])
                    else:
                        strings[string[0]] = string[1]
    for dirdata in os.walk(OSPFMSOURCEDIR):
        for filename in dirdata[2]:
            if filename.endswith('.py'):
                for string in get_ospfm_api_error_strings(
                                    os.path.join(dirdata[0], filename)
                                ):
                    if string in strings:
                        strings[string].append(
                            'OSPFM error message in {}'.format(filename)
                        )
                    else:
                        strings[string] = [
                            'OSPFM error message in {}'.format(filename)
                        ]
    return strings



def get_strings(filename):
    strings = {}
    with open(filename, 'r') as f:
        for linenb, line in enumerate(f.readlines()):
            for string in re.findall("_\('([^\']*)'\)", line):
                if string in strings:
                    strings[string].append('%s:%d' % (filename, linenb))
                else:
                    strings[string] = ['%s:%d' % (filename, linenb)]
    return strings



def get_ospfm_api_error_strings(filename):
    content = open(filename, 'r').read()
    content = re.sub('\( *\n *', '(', content, flags=re.MULTILINE)
    return re.findall("self.badrequest\('([^\'\)]*)'\)", content) + \
           re.findall('self.badrequest\("([^\"\)]*)"\)', content) + \
           re.findall("self.forbidden\('([^\'\)]*)'\)", content) + \
           re.findall('self.forbidden\("([^\"\)]*)"\)', content) + \
           re.findall("self.notfound\('([^\'\)]*)'\)", content) + \
           re.findall('self.notfound\("([^\"\)]*)"\)', content)



def update_all_translation_files(strings):
    locales = []
    files = []
    for filename in os.listdir(TRANSLATIONDIR):
        if filename.endswith('.js'):
            files.append(filename)
            locales.append(filename[:-3])
    targetdir = tempfile.mkdtemp()
    for filename in files:
        update_translation_file(
            filename,
            targetdir,
            strings,
            locales
        )
    shutil.rmtree(targetdir)



def entries(name, currentvalues, defaultvalues):
    if name in currentvalues:
        already_defined = currentvalues[name]
{'fraction_non_alphanumeric': 0.23385254413291798, 'fraction_numerical': 0.004153686396677051, 'mean_word_length': 2.6111736032995876, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5594068', 'n_tokens_mistral': 3506, 'n_tokens_neox': 3365, 'n_words': 888}