<commit_before>const flatten = require('flat');

module.exports = function (inputJson, options) {
    const pathRegex = new RegExp(options.includePaths);
    inputJson = JSON.parse(inputJson);
    const paths = inputJson.paths;

    let definitionsMap = {};
    for (const path in paths) {
        searchReferencesFor(paths[path], inputJson, definitionsMap);
    }
    removeUnwantedKeys(paths, pathRegex);
    let whiteList = {};
    for (const path in paths) {
        searchReferencesFor(paths[path], inputJson, whiteList);
    }

    clearMismatchedElements(inputJson, definitionsMap, whiteList);
    return filterJson(inputJson, whiteList);
};

function removeUnwantedKeys(objectToFilter, keyRegex) {
    for (const key in objectToFilter) {
        if (!keyRegex.test(key)) {
            delete objectToFilter[key];
        }
    }
}

function isObject(value) {
    return (typeof value === 'object');
}

function saveReference(defLocalizationName, defName, definitionsMap) {
    definitionsMap[defLocalizationName] = definitionsMap[defLocalizationName] || [];
    definitionsMap[defLocalizationName].push(defName);
}

function shouldSaveReference(defLocalizationName, defName, definitionsMap) {
    return !definitionsMap[defLocalizationName] || definitionsMap[defLocalizationName].indexOf(defName) < 0;
}

function searchReferencesFor(element, inputJson, definitionsMap) {
    const flattenElement = flatten(element);
    for (const key in flattenElement) {
        let value = flattenElement[key];
        if (key.includes('$ref')) {
            const [defLocalizationName, defName] = value.slice(2).split('/');
            const next = inputJson[defLocalizationName][defName];
            if (shouldSaveReference(defLocalizationName, defName, definitionsMap)) {
                saveReference(defLocalizationName, defName, definitionsMap);
                if (isObject(next)) {
                    searchReferencesFor(next, inputJson, definitionsMap);

                }
            }
        }
    }
}

function clearMismatchedElements(obj, someList, whiteList) {
    for (const key in someList) {
        if (!whiteList.hasOwnProperty(key)) {
            obj[key] = {};
        }
    }
}

function filterJson(inputJson, definitionsMap) {
    for (const defLocalizationName in definitionsMap) {
        let expectedDefNames = definitionsMap[defLocalizationName];
        for (const key in inputJson[defLocalizationName]) {
            if (expectedDefNames.indexOf(key) < 0) {
                delete inputJson[defLocalizationName][key];
            }
        }
    }
    return JSON.stringify(inputJson);
}

<commit_msg>Rename variables in searchReferencesFor method.<commit_after>const flatten = require('flat');

module.exports = function (inputJson, options) {
    const pathRegex = new RegExp(options.includePaths);
    inputJson = JSON.parse(inputJson);
    const paths = inputJson.paths;

    let definitionsMap = {};
    for (const path in paths) {
        searchReferencesFor(paths[path], inputJson, definitionsMap);
    }
    removeUnwantedKeys(paths, pathRegex);
    let whiteList = {};
    for (const path in paths) {
        searchReferencesFor(paths[path], inputJson, whiteList);
    }

    clearMismatchedElements(inputJson, definitionsMap, whiteList);
    return filterJson(inputJson, whiteList);
};

function removeUnwantedKeys(objectToFilter, keyRegex) {
    for (const key in objectToFilter) {
        if (!keyRegex.test(key)) {
            delete objectToFilter[key];
        }
    }
}

function isObject(value) {
    return (typeof value === 'object');
}

function saveReference(defLocalizationName, defName, definitionsMap) {
    definitionsMap[defLocalizationName] = definitionsMap[defLocalizationName] || [];
    definitionsMap[defLocalizationName].push(defName);
}

function shouldSaveReference(defLocalizationName, defName, definitionsMap) {
    return !definitionsMap[defLocalizationName] || definitionsMap[defLocalizationName].indexOf(defName) < 0;
}

function searchReferencesFor(definition, inputJson, definitionsMap) {
    const flattenDefinition = flatten(definition);
    for (const key in flattenDefinition) {
        let value = flattenDefinition[key];
        if (key.includes('$ref')) {
            const [defLocalizationName, defName] = value.slice(2).split('/');
            const nestedDefinition = inputJson[defLocalizationName][defName];
            if (shouldSaveReference(defLocalizationName, defName, definitionsMap)) {
                saveReference(defLocalizationName, defName, definitionsMap);
                if (isObject(nestedDefinition)) {
                    searchReferencesFor(nestedDefinition, inputJson, definitionsMap);
                }
            }
        }
    }
}

function clearMismatchedElements(obj, someList, whiteList) {
    for (const key in someList) {
        if (!whiteList.hasOwnProperty(key)) {
            obj[key] = {};
        }
    }
}

function filterJson(inputJson, definitionsMap) {
    for (const defLocalizationName in definitionsMap) {
        let expectedDefNames = definitionsMap[defLocalizationName];
        for (const key in inputJson[defLocalizationName]) {
            if (expectedDefNames.indexOf(key) < 0) {
                delete inputJson[defLocalizationName][key];
            }
        }
    }
    return JSON.stringify(inputJson);
}


{'fraction_non_alphanumeric': 0.10393258426966293, 'fraction_numerical': 0.0011235955056179776, 'mean_word_length': 3.2968624296057922, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4686583', 'n_tokens_mistral': 1463, 'n_tokens_neox': 1435, 'n_words': 300}