<commit_before>var tagDB = db.getSiblingDB('german_tag_graph');
var graphDB =  db.getSiblingDB('graph');

graphDB.nodes.ensureIndex({string: 1, language: 1});
graphDB.nodes.ensureIndex({language: 1, string: 1});
graphDB.edges.ensureIndex({type: 1, source: 1});
graphDB.edges.ensureIndex({source: 1});

var scripts = [
    'integration/tags.js',
    'integration/languageDetection.js',
    'integration/clicks.js'
];

scripts.forEach(function(script_path) {
    print(script_path.split('/').pop().slice(0, -3));
    load(script_path);
});<commit_msg>spice things up with unique indexes ;)<commit_after>var tagDB = db.getSiblingDB('german_tag_graph');
var graphDB =  db.getSiblingDB('graph');

graphDB.nodes.ensureIndex({language: 1, string: 1}, {unique: true});
graphDB.nodes.ensureIndex({string: 1, language: 1});
graphDB.edges.ensureIndex({type: 1, source: 1}, {unique: true});
graphDB.edges.ensureIndex({source: 1});

var scripts = [
    'integration/tags.js',
    'integration/languageDetection.js',
    'integration/clicks.js'
];

scripts.forEach(function(script_path) {
    print(script_path.split('/').pop().slice(0, -3));
    load(script_path);
});
{'fraction_non_alphanumeric': 0.18354978354978355, 'fraction_numerical': 0.015584415584415584, 'mean_word_length': 5.682080924855492, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7225119', 'n_tokens_mistral': 414, 'n_tokens_neox': 395, 'n_words': 68}