<commit_before>// join with tag_link_object
var map = function() {
    emit (
        this.tag_id,
        {
            o:[{
                ot: this.object_type_id,
                oid: this.object_id
            }]
        }
    );
}

var reduce = function(key, values) {

    var r = {t: null, l: null, o: []};

    values.forEach(function(value) {
        if (value.t) {
            r.t = value.t;
        }
        if (value.l) {
            r.l = value.l
        }
        if (value.o && value.o.length > 0) {
            r.o = r.o.concat(value.o);
        }
    });
    return r;
}

db.ss_tag_link_object.mapReduce(map, reduce, {out: {reduce: "mr_tags", sharded: true}});

// remove unreferenced and null tags
db.mr_tags.remove({"value.t": null});
db.mr_tags.remove({"value.l": null});
db.mr_tags.remove({"value.o": {$size: 0}});<commit_msg>also remove tags without links<commit_after>// join with tag_link_object
var map = function() {
    emit (
        this.tag_id,
        {
            o:[{
                ot: this.object_type_id,
                oid: this.object_id
            }]
        }
    );
}

var reduce = function(key, values) {

    var r = {t: null, l: null, o: []};

    values.forEach(function(value) {
        if (value.t) {
            r.t = value.t;
        }
        if (value.l) {
            r.l = value.l
        }
        if (value.o && value.o.length > 0) {
            r.o = r.o.concat(value.o);
        }
    });
    return r;
}

db.ss_tag_link_object.mapReduce(map, reduce, {out: {reduce: "mr_tags", sharded: true}});

// remove unreferenced and null tags
db.mr_tags.remove({"value.t": null});
db.mr_tags.remove({"value.l": null});
db.mr_tags.remove({"value.o": {$size: 0}});
db.mr_tags.remove({"value.o":{$not: {$exists:1}}});
{'fraction_non_alphanumeric': 0.17374080362195812, 'fraction_numerical': 0.0028296547821165816, 'mean_word_length': 1.7886435331230284, 'pattern_counts': {'":': 7, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6014756', 'n_tokens_mistral': 654, 'n_tokens_neox': 635, 'n_words': 127}