<commit_before>var _ = require('lodash');
var GraphQL = require('./GraphQL');
var GraphQLFilter = require('./GraphQLFilter');
var Util = require('./Util');

var MetaStore = function() {
    this._rawData = {};
};

MetaStore.prototype.export = function () {
    return JSON.stringify(this._rawData, null, 2);
}

MetaStore.prototype.import = function (data) {
    this._rawData = JSON.parse(data);
    this._resetCompile();
}

MetaStore.prototype.addData = function (extractorName, fileName, data) {
    if (this._rawData[extractorName] === undefined) {
        this._rawData[extractorName] = {};
    }
    this._rawData[extractorName][fileName] = data;
    this._resetCompile();
};

MetaStore.prototype.getData = function (extractorName, fileName) {
    if (!this._rawData[extractorName]) return []
    return this._rawData[extractorName][fileName] || [];
};

MetaStore.prototype.getExtractors = function () {
    return Object.keys(this._rawData);
};

MetaStore.prototype.getFiles = function () {
    var fileNameMap = {};
    for (var key in this._rawData) {
        for (var fileName in this._rawData[key]) {
            fileNameMap[fileName] = true;
        }
    }
    return Object.keys(fileNameMap);
};

MetaStore.prototype.removeExtractor = function (extractorName) {
    delete this._rawData[extractorName];
    this._resetCompile();
};

MetaStore.prototype.removeFile = function (fileName) {
    for (var key in this._rawData) {
        delete this._rawData[key][fileName];
    }
    this._resetCompile();
};

MetaStore.prototype.query = function (queryString, isParsed) {
    var compiled = this._getCompiled();
    var query = isParsed ? queryString : GraphQL.parse(queryString);
    return GraphQLFilter.filter(compiled[query.getType()], query);
};

MetaStore.prototype._getCompiled = function () {
    if (this._compiled === undefined) {
        this._compiled = Util.compressData(this._rawData);
    }<commit_msg>don't return __metadata in keys<commit_after>var _ = require('lodash');
var GraphQL = require('./GraphQL');
var GraphQLFilter = require('./GraphQLFilter');
var Util = require('./Util');

var MetaStore = function() {
    this._rawData = {};
};

MetaStore.prototype.export = function () {
    return JSON.stringify(this._rawData, null, 2);
}

MetaStore.prototype.import = function (data) {
    this._rawData = JSON.parse(data);
    this._resetCompile();
}

MetaStore.prototype.addData = function (extractorName, fileName, data) {
    if (this._rawData[extractorName] === undefined) {
        this._rawData[extractorName] = {};
    }
    this._rawData[extractorName][fileName] = data;
    this._resetCompile();
};

MetaStore.prototype.getData = function (extractorName, fileName) {
    if (!this._rawData[extractorName]) return []
    return this._rawData[extractorName][fileName] || [];
};

MetaStore.prototype.getExtractors = function () {
    return Object.keys(this._rawData);
};

MetaStore.prototype.getFiles = function () {
    var fileNameMap = {};
    for (var key in this._rawData) {
        for (var fileName in this._rawData[key]) {
            if (fileName !== '__metadata') {
                fileNameMap[fileName] = true;
            }
        }
    }
    return Object.keys(fileNameMap);
};

MetaStore.prototype.removeExtractor = function (extractorName) {
    delete this._rawData[extractorName];
    this._resetCompile();
};

MetaStore.prototype.removeFile = function (fileName) {
    for (var key in this._rawData) {
        delete this._rawData[key][fileName];
    }
    this._resetCompile();
};

MetaStore.prototype.query = function (queryString, isParsed) {
    var compiled = this._getCompiled();
    var query = isParsed ? queryString : GraphQL.parse(queryString);
    return GraphQLFilter.filter(compiled[query.getType()], query);
};

MetaStore.prototype._getCompiled = function () {
    if (this._compiled === undefined) {
        this._compiled = Util.compressData(this._rawData);
    }
{'fraction_non_alphanumeric': 0.14282083862770012, 'fraction_numerical': 0.0005082592121982211, 'mean_word_length': 3.725090036014406, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1522413', 'n_tokens_mistral': 1181, 'n_tokens_neox': 1188, 'n_words': 225}