<commit_before>ret = db.feeds.createIndex({id: 1}, {unique: true }); checkret('feeds id', ret);

ret = db.labels.createIndex({ "semanticId" : 1 }, { unique: true }); checkret('labels semanticId', ret);
ret = db.labels.createIndex({ "when" : 1 }); checkret('labels when', ret);

ret = db.metadata.createIndex({ "semanticId" : 1 }); checkret('metadata semanticId', ret);
ret = db.metadata.createIndex({ "userId" : 1 }); checkret('metadata userId', ret);
ret = db.metadata.createIndex({ "semantic" : 1 }); checkret('metadata semantic', ret);
ret = db.metadata.createIndex({ "id" : 1 }, { unique : true }); checkret('metadata id', ret);
ret = db.metadata.createIndex({ "impressionTime" : -1 }); checkret('metadata impressionTime', ret);

ret = db.semantics.createIndex({ "label" : 1 });
ret = db.semantics.createIndex({ "when" : 1 });
ret = db.semantics.createIndex({ "semanticId" : 1 });

ret = db.parsererrors.createIndex({ "id": 1 }); checkret('parsererror id', ret);

ret = db.aggregated.createIndex({ "hourId": 1 }, {unique: true}); checkret('aggregated hourId id', ret);

ret = db.summary.createIndex({ "id": 1 }, { unique: true }); checkret('summary id', ret);
ret = db.summary.createIndex({ "impressionTime": -1 }); checkret('summary id', ret);

function checkret(info, retval) {
    retval.info = info;
    printjson(retval);
};<commit_msg>nested index for global post count<commit_after>ret = db.feeds.createIndex({id: 1}, {unique: true }); checkret('feeds id', ret);

ret = db.labels.createIndex({ "semanticId" : 1 }, { unique: true }); checkret('labels semanticId', ret);
ret = db.labels.createIndex({ "when" : 1 }); checkret('labels when', ret);

ret = db.metadata.createIndex({ "semanticId" : 1 }); checkret('metadata semanticId', ret);
ret = db.metadata.createIndex({ "userId" : 1 }); checkret('metadata userId', ret);
ret = db.metadata.createIndex({ "semantic" : 1 }); checkret('metadata semantic', ret);
ret = db.metadata.createIndex({ "id" : 1 }, { unique : true }); checkret('metadata id', ret);
ret = db.metadata.createIndex({ "impressionTime" : -1 }); checkret('metadata impressionTime', ret);
ret = db.metadata.createIndex({ "linkedtime.postId" : 1 }); checkret('metadata linkedtime.postId', ret);

ret = db.semantics.createIndex({ "label" : 1 });
ret = db.semantics.createIndex({ "when" : 1 });
ret = db.semantics.createIndex({ "semanticId" : 1 });

ret = db.parsererrors.createIndex({ "id": 1 }); checkret('parsererror id', ret);

ret = db.aggregated.createIndex({ "hourId": 1 }, {unique: true}); checkret('aggregated hourId id', ret);

ret = db.summary.createIndex({ "id": 1 }, { unique: true }); checkret('summary id', ret);
ret = db.summary.createIndex({ "impressionTime": -1 }); checkret('summary id', ret);

function checkret(info, retval) {
    retval.info = info;
    printjson(retval);
};
{'fraction_non_alphanumeric': 0.19992897727272727, 'fraction_numerical': 0.011008522727272728, 'mean_word_length': 5.431506849315069, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2336079', 'n_tokens_mistral': 929, 'n_tokens_neox': 897, 'n_words': 236}