<commit_before>// build edges for mcl

db.mcl_edges.drop()
db.mcl_edges.ensureIndex({a: 1, b: 1})
db.tag_graph_edges.find().addOption(DBQuery.Option.noTimeout).forEach(function(edge) {
    edge.tn.forEach(function(t) {
        if (db.mcl_edges.count({$or: [{a: edge._id, b: t.id}, {a: t.id, b: edge._id}]}) == 0) {
            db.mcl_edges.insert({
                a: edge._id,
                b: t.id,
                c: t.ochiai
            })
        }
    })
})<commit_msg>mcl: include all measurements and use tags as labels<commit_after>// build edges for mcl

db.mcl_edges.drop()
db.mcl_edges.ensureIndex({a: 1, b: 1})
db.tag_graph_edges.find().addOption(DBQuery.Option.noTimeout).forEach(function(edge) {
    var a = db.tag_graph_nodes.findOne({_id: edge._id}).tag
    edge.tn.forEach(function(t) {
        var b = db.tag_graph_nodes.findOne({_id: t._id}).tag
        if (db.mcl_edges.count({$or: [{a: a, b: b}, {a: b, b: a}]}) == 0) {
            db.mcl_edges.insert({
                a: a,
                b: b,
                c: t.dice
            })
        }
    })
})
{'fraction_non_alphanumeric': 0.17467652495378927, 'fraction_numerical': 0.005545286506469501, 'mean_word_length': 2.0854700854700856, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3716590', 'n_tokens_mistral': 418, 'n_tokens_neox': 405, 'n_words': 76}