<commit_before>var Path = require('path'),
    fs = require('fs'),
    _ = require('underscore'),
    passError = require('passerror'),
    memoizeAsync = require('./memoizeAsync'),
    libxmljs = require('libxmljs'),
    seq = require('seq'),
    normalizeLocaleId = require('./normalizeLocaleId'),
    convertObjectsWithIntegerKeysToArrays = require('./convertObjectsWithIntegerKeysToArrays'),
    cldrPluralRuleToJavaScriptAst = require('./cldrPluralRuleToJavaScriptAst'),
    CldrRbnfRuleSet = require('./CldrRbnfRuleSet'),
    uglifyJs = require('uglify-js');

function normalizeXPathQuery(xpathQuery) {
    var xpathQueryFragments = xpathQuery.split('/');
    for (var i = 0 ; i < xpathQueryFragments.length ; i += 1) {
        if (i > 0 && xpathQueryFragments[i] === '..' && xpathQueryFragments[i - 1] !== '..') {
            xpathQueryFragments.splice(i - 1, 2);
            i -= 2;
        }
    }
    return xpathQueryFragments.join('/');
}

function expandLocaleIdToPrioritizedList(localeId) {
    if (!localeId) {
        return [];
    }
    var localeIds = [localeId];
    while (/_[^_]+$/.test(localeId)) {
        localeId = localeId.replace(/_[^_]+$/, '');
        localeIds.push(localeId);
    }
    return localeIds;
}

function Cldr(cldrPath) {
    // Support instantiation without the 'new' operator:
    if (!(this instanceof Cldr)) {
        return new Cldr(cldrPath);
    }
    this.cldrPath = cldrPath;
    this.documentByFileName = {};
    this.fileNamesByTypeAndNormalizedLocaleId = {};

    // Sorry, but it's just 4 sync calls per Cldr instance:
    ['main', 'rbnf'].forEach(function (type) {
        this.fileNamesByTypeAndNormalizedLocaleId[type] = {};
        var fileNames;
        try {
            fileNames = fs.readdirSync(Path.resolve(this.cldrPath, "common", type));
        } catch (e) {
            if (e.code === 'ENOENT') {
                // Directory doesn't exist, just pretend it's empty.
                return;
            }
        }
        fileNames.forEach(function (fileName) {
            var matchFileName = fileName.match(/^(.*)\.xml$/);
            if (matchFileName) {
                this.fileNamesByTypeAndNormalizedLocaleId[type][normalizeLocaleId(matchFileName[1])] =
                    Path.resolve(this.cldrPath, "common", type, fileName);
            }
        }, this);
    }, this);

    this.localeIds = Object.keys(this.fileNamesByTypeAndNormalizedLocaleId.main);
    this.memoizerByFileName = {};
}

Cldr.prototype = {
    get calendarIds() {
        if (!this._calendarIds) {
            this._calendarIds = [];
            this.getDocument(Path.resolve(this.cldrPath, 'common', 'bcp47', 'calendar.xml')).find('/ldmlBCP47/keyword/key[@name="ca"]/type').forEach(function (keyNode) {
                this._calendarIds.push(keyNode.attr('name').value());
            }, this);
        }
        return this._calendarIds;
    },

    get numberSystemIds() {
        if (!this._numberSystemIds) {
            this._numberSystemIds = [];
            this.getDocument(Path.resolve(this.cldrPath, 'common', 'bcp47', 'number.xml')).find('/ldmlBCP47/keyword/key[@name="nu"]/type').forEach(function (keyNode) {
                this._numberSystemIds.push(keyNode.attr('name').value());
            }, this);
        }
        return this._numberSystemIds;
    },

    // Works both async and sync (omit cb):
    getDocument: function (fileName, cb) {
        var that = this;
        if (that.documentByFileName[fileName]) {
            if (cb) {
                process.nextTick(function () {
                    cb(null, that.documentByFileName[fileName]);
                });
            } else {
                return that.documentByFileName[fileName];
            }
        } else {
            if (cb) {
                // Make sure not to load file more than once if it's being loaded when the getDocument is called for the second time:
                that.memoizerByFileName[fileName] = that.memoizerByFileName[fileName] || memoizeAsync(function (cb) {
                    fs.readFile(fileName, 'utf-8', passError(cb, function (xmlString) {
                        var document = libxmljs.parseXmlString(xmlString);
                        that.documentByFileName[fileName] = document;
                        cb(null, document);
                    }));
                });
                that.memoizerByFileName[fileName](cb);
            } else {
                return that.documentByFileName[fileName] = libxmljs.parseXmlString(fs.readFileSync(fileName, 'utf-8'));
            }
        }
    },

    getPrioritizedDocumentsForLocale: function (localeId, type) {
        var that = this;
        return expandLocaleIdToPrioritizedList(localeId).concat('root').map(function (subLocaleId) {
            return that.fileNamesByTypeAndNormalizedLocaleId[type][normalizeLocaleId(subLocaleId)];
        }).filter(function (fileName) {
            return !!fileName;
        }).map(function (fileName) {
            return that.getDocument(fileName);
        });
    },

    load: function (localeIds, cb) {
        var that = this;
        if (typeof localeIds === 'function') {
            cb = localeIds;
            localeIds = that.localeIds;
        }
        localeIds = (Array.isArray(localeIds) ? localeIds : [localeIds]).map(normalizeLocaleId);
        var neededLocaleById = {root: true};
        localeIds.forEach(function (localeId) {
            expandLocaleIdToPrioritizedList(localeId).forEach(function (subLocaleId) {
                neededLocaleById[subLocaleId] = true;
            });
        });
        var fileNames = [
            Path.resolve(that.cldrPath, 'common', 'supplemental', 'plurals.xml'),
            Path.resolve(that.cldrPath, 'common', 'supplemental', 'numberingSystems.xml')
        ];
        Object.keys(neededLocaleById).forEach(function (localeId) {
            ['main', 'rbnf'].forEach(function (type) {
                var fileName = that.fileNamesByTypeAndNormalizedLocaleId[type][localeId];
                if (fileName) {
                    fileNames.push(fileName);
                }
            });
        });
        seq(fileNames)
            .parEach(20, function (fileName) {
                that.getDocument(fileName, this);
            })
            .seq(function () {
                cb();
            })
            .catch(cb);
    },

    createFinder: function (prioritizedDocuments) {
        return function finder(xpathQuery) {
            var prioritizedResults = [];
            prioritizedDocuments.forEach(function (document, i) {
                var resultsForLocaleDocument = document.find(xpathQuery);
                if (resultsForLocaleDocument.length === 0 && i === (prioritizedDocuments.length - 1)) {
                    // We're in root and there were no results, look for alias elements in path:
                    var queryFragments = xpathQuery.split('/'),
                        poppedQueryFragments = [];
                    while (queryFragments.length > 1) {
                        var aliasNodes = document.find(queryFragments.join('/') + '/alias');
                        if (aliasNodes.length > 0) {
                            var aliasSpecifiedQuery = normalizeXPathQuery(aliasNodes[0].path() + '/../' + aliasNodes[0].attr('path').value() + '/' + poppedQueryFragments.join('/'));
                            Array.prototype.push.apply(prioritizedResults, finder(aliasSpecifiedQuery));
                            break;
                        }
                        poppedQueryFragments.unshift(queryFragments.pop());
                    }
                } else {
                    Array.prototype.push.apply(prioritizedResults, resultsForLocaleDocument);
                }
            });
            return prioritizedResults;
        };
    },

    extractLanguageDisplayNames: function (localeId) {
        var finder = this.createFinder(this.getPrioritizedDocumentsForLocale(localeId, 'main')),
            languageDisplayNames = {};<commit_msg>Use the original query instead of aliasNode.path() when building the query for following the alias. Fixes #1.<commit_after>var Path = require('path'),
    fs = require('fs'),
    _ = require('underscore'),
    passError = require('passerror'),
    memoizeAsync = require('./memoizeAsync'),
    libxmljs = require('libxmljs'),
    seq = require('seq'),
    normalizeLocaleId = require('./normalizeLocaleId'),
    convertObjectsWithIntegerKeysToArrays = require('./convertObjectsWithIntegerKeysToArrays'),
    cldrPluralRuleToJavaScriptAst = require('./cldrPluralRuleToJavaScriptAst'),
    CldrRbnfRuleSet = require('./CldrRbnfRuleSet'),
    uglifyJs = require('uglify-js');

function normalizeXPathQuery(xpathQuery) {
    var xpathQueryFragments = xpathQuery.split('/');
    for (var i = 0 ; i < xpathQueryFragments.length ; i += 1) {
        if (i > 0 && xpathQueryFragments[i] === '..' && xpathQueryFragments[i - 1] !== '..') {
            xpathQueryFragments.splice(i - 1, 2);
            i -= 2;
        }
    }
    return xpathQueryFragments.join('/');
}

function expandLocaleIdToPrioritizedList(localeId) {
    if (!localeId) {
        return [];
    }
    var localeIds = [localeId];
    while (/_[^_]+$/.test(localeId)) {
        localeId = localeId.replace(/_[^_]+$/, '');
        localeIds.push(localeId);
    }
    return localeIds;
}

function Cldr(cldrPath) {
    // Support instantiation without the 'new' operator:
    if (!(this instanceof Cldr)) {
        return new Cldr(cldrPath);
    }
    this.cldrPath = cldrPath;
    this.documentByFileName = {};
    this.fileNamesByTypeAndNormalizedLocaleId = {};

    // Sorry, but it's just 4 sync calls per Cldr instance:
    ['main', 'rbnf'].forEach(function (type) {
        this.fileNamesByTypeAndNormalizedLocaleId[type] = {};
        var fileNames;
        try {
            fileNames = fs.readdirSync(Path.resolve(this.cldrPath, "common", type));
        } catch (e) {
            if (e.code === 'ENOENT') {
                // Directory doesn't exist, just pretend it's empty.
                return;
            }
        }
        fileNames.forEach(function (fileName) {
            var matchFileName = fileName.match(/^(.*)\.xml$/);
            if (matchFileName) {
                this.fileNamesByTypeAndNormalizedLocaleId[type][normalizeLocaleId(matchFileName[1])] =
                    Path.resolve(this.cldrPath, "common", type, fileName);
            }
        }, this);
    }, this);

    this.localeIds = Object.keys(this.fileNamesByTypeAndNormalizedLocaleId.main);
    this.memoizerByFileName = {};
}

Cldr.prototype = {
    get calendarIds() {
        if (!this._calendarIds) {
            this._calendarIds = [];
            this.getDocument(Path.resolve(this.cldrPath, 'common', 'bcp47', 'calendar.xml')).find('/ldmlBCP47/keyword/key[@name="ca"]/type').forEach(function (keyNode) {
                this._calendarIds.push(keyNode.attr('name').value());
            }, this);
        }
        return this._calendarIds;
    },

    get numberSystemIds() {
        if (!this._numberSystemIds) {
            this._numberSystemIds = [];
            this.getDocument(Path.resolve(this.cldrPath, 'common', 'bcp47', 'number.xml')).find('/ldmlBCP47/keyword/key[@name="nu"]/type').forEach(function (keyNode) {
                this._numberSystemIds.push(keyNode.attr('name').value());
            }, this);
        }
        return this._numberSystemIds;
    },

    // Works both async and sync (omit cb):
    getDocument: function (fileName, cb) {
        var that = this;
        if (that.documentByFileName[fileName]) {
            if (cb) {
                process.nextTick(function () {
                    cb(null, that.documentByFileName[fileName]);
                });
            } else {
                return that.documentByFileName[fileName];
            }
        } else {
            if (cb) {
                // Make sure not to load file more than once if it's being loaded when the getDocument is called for the second time:
                that.memoizerByFileName[fileName] = that.memoizerByFileName[fileName] || memoizeAsync(function (cb) {
                    fs.readFile(fileName, 'utf-8', passError(cb, function (xmlString) {
                        var document = libxmljs.parseXmlString(xmlString);
                        that.documentByFileName[fileName] = document;
                        cb(null, document);
                    }));
                });
                that.memoizerByFileName[fileName](cb);
            } else {
                return that.documentByFileName[fileName] = libxmljs.parseXmlString(fs.readFileSync(fileName, 'utf-8'));
            }
        }
    },

    getPrioritizedDocumentsForLocale: function (localeId, type) {
        var that = this;
        return expandLocaleIdToPrioritizedList(localeId).concat('root').map(function (subLocaleId) {
            return that.fileNamesByTypeAndNormalizedLocaleId[type][normalizeLocaleId(subLocaleId)];
        }).filter(function (fileName) {
            return !!fileName;
        }).map(function (fileName) {
            return that.getDocument(fileName);
        });
    },

    load: function (localeIds, cb) {
        var that = this;
        if (typeof localeIds === 'function') {
            cb = localeIds;
            localeIds = that.localeIds;
        }
        localeIds = (Array.isArray(localeIds) ? localeIds : [localeIds]).map(normalizeLocaleId);
        var neededLocaleById = {root: true};
        localeIds.forEach(function (localeId) {
            expandLocaleIdToPrioritizedList(localeId).forEach(function (subLocaleId) {
                neededLocaleById[subLocaleId] = true;
            });
        });
        var fileNames = [
            Path.resolve(that.cldrPath, 'common', 'supplemental', 'plurals.xml'),
            Path.resolve(that.cldrPath, 'common', 'supplemental', 'numberingSystems.xml')
        ];
        Object.keys(neededLocaleById).forEach(function (localeId) {
            ['main', 'rbnf'].forEach(function (type) {
                var fileName = that.fileNamesByTypeAndNormalizedLocaleId[type][localeId];
                if (fileName) {
                    fileNames.push(fileName);
                }
            });
        });
        seq(fileNames)
            .parEach(20, function (fileName) {
                that.getDocument(fileName, this);
            })
            .seq(function () {
                cb();
            })
            .catch(cb);
    },

    createFinder: function (prioritizedDocuments) {
        return function finder(xpathQuery) {
            var prioritizedResults = [];
            prioritizedDocuments.forEach(function (document, i) {
                var resultsForLocaleDocument = document.find(xpathQuery);
                if (resultsForLocaleDocument.length === 0 && i === (prioritizedDocuments.length - 1)) {
                    // We're in root and there were no results, look for alias elements in path:
                    var queryFragments = xpathQuery.split('/'),
                        poppedQueryFragments = [];
                    while (queryFragments.length > 1) {
                        var aliasNodes = document.find(queryFragments.join('/') + '/alias');
                        if (aliasNodes.length > 0) {
                            var aliasSpecifiedQuery = normalizeXPathQuery(queryFragments.join('/') + '/' + aliasNodes[0].attr('path').value() + '/' + poppedQueryFragments.join('/'));
                            Array.prototype.push.apply(prioritizedResults, finder(aliasSpecifiedQuery));
                            break;
                        }
                        poppedQueryFragments.unshift(queryFragments.pop());
                    }
                } else {
                    Array.prototype.push.apply(prioritizedResults, resultsForLocaleDocument);
                }
            });
            return prioritizedResults;
        };
    },

    extractLanguageDisplayNames: function (localeId) {
        var finder = this.createFinder(this.getPrioritizedDocumentsForLocale(localeId, 'main')),
            languageDisplayNames = {};
{'fraction_non_alphanumeric': 0.12748559747258875, 'fraction_numerical': 0.003345103140680171, 'mean_word_length': 2.047187617969045, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2036078', 'n_tokens_mistral': 4570, 'n_tokens_neox': 4421, 'n_words': 784}