<commit_before>var EN = require("../../www/common/translations/messages.json");

var simpleTags = [
    '<br>',
    '<a href="/login/">',
    '<a href="/register/">',

    // FIXME
    "<a href='#'>",
    '<a href="#docs">',
    '<h3>',
    '</h3>',

    // FIXME register_notes
    '<ul class="cp-notes-list">',
    '</ul>',
    '<span class="red">',
    '</span>',
];

['a', 'b', 'em',/* 'p',*/ 'i', 'code', 'li',].forEach(function (tag) {
    simpleTags.push('<' + tag + '>');
    simpleTags.push('</' + tag + '>');
});

var found_tags = {};

// these keys are known to be problematic
var KNOWN_ISSUES = [ // FIXME
    //'newVersion',
    //'fm_info_anonymous',
    //'register_notes',
];

var special_rules = {};

special_rules.en = function (s) {
    // Prefer the american -ize suffix for verbs rather than -ise
    return /[^w]ise(\s|$)/.test(s);
};

special_rules.fr = function (s) {
/*
    hacky regexp to check whether there are any instances of ':'
    which do not have the preceding space as is expected.
    ignore instances where the following character is a '/'
    because this is probably a URL (http(s)://)
*/
    return /\S[:;\?\!][^\/]{1,}/.test(s);
};

var noop = function () {};

var processLang = function (map, lang, primary) {
    var announced = false;
    var announce = function () {
        if (announced) { return; }
        announced = true;
        console.log("NEXT LANGUAGE: ", lang);
    };

    var special = special_rules[lang] || noop;
    Object.keys(map).forEach(function (k) {
        if (!EN[k]) { return; }
        if (KNOWN_ISSUES.indexOf(k) !== -1) { return; }

        var s = map[k];
        if (typeof(s) !== 'string') { return; }
        var usesHTML;

        s.replace(/<[\s\S]*?>/g, function (html) {
            if (simpleTags.includes(html)) {
                found_tags[html] = 1;
                return;
            }
            announce();
            usesHTML = true;
            if (!primary) {
                console.log("{%s}", html);
            }
        });

        var weirdCapitalization;
        s.replace(/cryptpad(\.fr|\.org)*/gi, function (brand) {
            if (['CryptPad', 'cryptpad.fr', 'cryptpad.org'].includes(brand)) { return; }
            weirdCapitalization = true;
        });

        var specialViolation = special(s);

        if (usesHTML || weirdCapitalization || specialViolation) {
            announce();
            console.log("%s", s);
            console.log("[%s]\n", k);
        }
    });
};

processLang(EN, 'en', true);

[
  'ar',
  //'bn_BD',
  'ca',
  'cs',
  'de',
  'es',
  'eu',
  'fi',
  'fr',
  'hi',
  'it',
  'ja',
  'nb',
  'nl',
  'pl',
  'pt-br',
  'ro',
  'ru',
  'sv',
  //'te',<commit_msg>add translation linter rules to compare markup against base translation<commit_after>var EN = require("../../www/common/translations/messages.json");
var Util = require("../../www/common/common-util.js");

var simpleTags = [
    '<br>',
    '<a href="/login/">',
    '<a href="/register/">',

    // FIXME
    "<a href='#'>",
    '<a href="#docs">',
    '<h3>',
    '</h3>',

    // FIXME register_notes
    '<ul class="cp-notes-list">',
    '</ul>',
    '<span class="red">',
    '</span>',
];

['a', 'b', 'em',/* 'p',*/ 'i', 'code', 'li',].forEach(function (tag) {
    simpleTags.push('<' + tag + '>');
    simpleTags.push('</' + tag + '>');
});

var found_tags = {};

// these keys are known to be problematic
var KNOWN_ISSUES = [ // FIXME
    //'newVersion',
    //'fm_info_anonymous',
    //'register_notes',
];

var special_rules = {};

special_rules.en = function (s) {
    // Prefer the american -ize suffix for verbs rather than -ise
    return /[^w]ise(\s|$)/.test(s);
};

special_rules.fr = function (s) {
/*
    hacky regexp to check whether there are any instances of ':'
    which do not have the preceding space as is expected.
    ignore instances where the following character is a '/'
    because this is probably a URL (http(s)://)
*/
    return /\S[:;\?\!][^\/]{1,}/.test(s);
};

var noop = function () {};

var getTags = S => {
    if (typeof(S) !== 'string') { return []; }

    var tags = [];
    S.replace(/<[\s\S]*?>/g, function (html) {
        tags.push(html);
    });
    return tags;
};

var getTagCount = T => {
    var M = {};
    T.forEach(html => {
        Util.inc(M, html);
    });
    return M;
};

// bidirectional comparison
var compareMarkup = (A, B) => {
    // if the frequency of some key in A does not match that of the same key in B
    if (Object.keys(A).some(k => {
        return A[k] !== B[k];
    })) {
        return false;
    }

    // same for B
    if (Object.keys(B).some(k => {
        return A[k] !== B[k];
    })) {
        return false;
    }

    return true;
};

var getReferenceMarkup = (function () {
    var O = {};
    return function (k) {
        var val = O[k];
        if (typeof(val) !== 'undefined') { return val; }
        var tags = getTags(EN[k]);
        val = O[k] = getTagCount(tags);
        return val;
    };
}());

var processLang = function (map, lang, primary) {
    var announced = false;
    var announce = function () {
        if (announced) { return; }
        announced = true;
        console.log("NEXT LANGUAGE: ", lang);
    };

    var special = special_rules[lang] || noop;
    Object.keys(map).forEach(function (k) {
        if (!EN[k]) { return; }
        if (KNOWN_ISSUES.indexOf(k) !== -1) { return; }

        var s = map[k];
        if (typeof(s) !== 'string') { return; }
        var usesHTML;

        var ref = getReferenceMarkup(k);
        var tags = getTags(s);
        var tagCount = getTagCount(tags);
        var markupMatches = compareMarkup(ref, tagCount);

        //console.log(ref);

        tags.forEach(html => {
            if (simpleTags.includes(html)) {
                found_tags[html] = 1;
                return;
            }
            announce();
            usesHTML = true;
            if (!primary) {
                console.log("{%s}", html);
            }
        });

        var weirdCapitalization;
        s.replace(/cryptpad(\.fr|\.org)*/gi, function (brand) {
            if (['CryptPad', 'cryptpad.fr', 'cryptpad.org'].includes(brand)) { return; }
            weirdCapitalization = true;
        });

        var specialViolation = special(s);

        if (usesHTML || weirdCapitalization || specialViolation || !markupMatches) {
            announce();
            console.log("%s", s);

            if (!markupMatches) {
                console.log("Markup does not match base translation");

                console.log("\nEnglish: \n\n%s\n", EN[k]);
                console.log(tagCount);
                console.log(ref);
            }
            //if (mismatchedTags.length) { console.log(mismatchedTags); } // XXX
            console.log("[%s]\n", k);
        }
    });
};

processLang(EN, 'en', true);

[
  'ar',
  //'bn_BD',
  'ca',
  'cs',
  'de',
  'es',
  'eu',
  'fi',
  'fr',
  'hi',
  'it',
  'ja',
  'nb',
  'nl',
  'pl',
  'pt-br',
  'ro',
  'ru',
  'sv',
  //'te',
{'fraction_non_alphanumeric': 0.19817897282686014, 'fraction_numerical': 0.0014226774790155072, 'mean_word_length': 2.0223559759243335, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1071650', 'n_tokens_mistral': 2404, 'n_tokens_neox': 2393, 'n_words': 565}