<commit_before>var async = require('async');

var _ = require('underscore');

var models = require('./models');

function mergeposts() {
    var processed_ids_set = {};
    var processed_ids_list = [];

    models.Post.find({'type': 'facebook'}, models.Post.PUBLIC_FIELDS).lean(true).exec(function (err, posts) {
        if (err) {
            console.error(err);
            process.exit(1);
            return;
        }

        // In series, otherwise we could have some funny race conditions (we have them anyway, but to a lesser extent)
        async.forEachSeries(posts, function (post, cb) {
            if (processed_ids_set[post.foreign_id]) {
                cb(null);
                return;
            }

            post = models.Post.cleanPost(post);

            models.Post.merge(post, function (err, post_merged, first_id, rest_ids) {
                if (err) {
                    console.log("Error while merging post (%s): %s", post.foreign_id, err);
                    cb(null);
                }
                else if (first_id && rest_ids) {
                    if (processed_ids_set[first_id]) throw new Error("Assertion failed");

                    processed_ids_set[first_id] = true;
                    processed_ids_list.push(first_id);
                    _.each(rest_ids, function (id, i, list) {
                        if (processed_ids_set[id]) throw new Error("Assertion failed");
                        processed_ids_set[id] = true;
                        processed_ids_list.push(id);
                    });

                    console.log("Merged Facebook posts: %s -> %s", rest_ids, first_id);
                    cb(null);
                }
                else {
                    models.Post.update({'type': 'facebook', 'foreign_id': post.foreign_id}, {'$unset': {'merged_to': true, 'merged_from': true}}, function (err, numberAffected, rawResponse) {
                        if (err) {
                            console.error("Error while setting post as not merged (%s): %s", post.foreign_id, err);
                        }
                        else if (numberAffected !== 1) {
                            console.error("Invalid number of Facebook posts set as not merged (" + post.foreign_id + ", " + numberAffected + "): " + rawResponse);
                        }
                        // We handle error independently
                        cb(null);
                    });
                }
            }, processed_ids_list);
        }, function (err) {
            process.exit(0);
        });
    });
}
<commit_msg>Use assert module for assertions.<commit_after>var async = require('async');
var assert = require('assert');

var _ = require('underscore');

var models = require('./models');

function mergeposts() {
    var processed_ids_set = {};
    var processed_ids_list = [];

    models.Post.find({'type': 'facebook'}, models.Post.PUBLIC_FIELDS).lean(true).exec(function (err, posts) {
        if (err) {
            console.error(err);
            process.exit(1);
            return;
        }

        // In series, otherwise we could have some funny race conditions (we have them anyway, but to a lesser extent)
        async.forEachSeries(posts, function (post, cb) {
            if (processed_ids_set[post.foreign_id]) {
                cb(null);
                return;
            }

            post = models.Post.cleanPost(post);

            models.Post.merge(post, function (err, post_merged, first_id, rest_ids) {
                if (err) {
                    console.log("Error while merging post (%s): %s", post.foreign_id, err);
                    cb(null);
                }
                else if (first_id && rest_ids) {
                    assert(!processed_ids_set[first_id]);

                    processed_ids_set[first_id] = true;
                    processed_ids_list.push(first_id);
                    _.each(rest_ids, function (id, i, list) {
                        assert(!processed_ids_set[id]);

                        processed_ids_set[id] = true;
                        processed_ids_list.push(id);
                    });

                    console.log("Merged Facebook posts: %s -> %s", rest_ids, first_id);
                    cb(null);
                }
                else {
                    models.Post.update({'type': 'facebook', 'foreign_id': post.foreign_id}, {'$unset': {'merged_to': true, 'merged_from': true}}, function (err, numberAffected, rawResponse) {
                        if (err) {
                            console.error("Error while setting post as not merged (%s): %s", post.foreign_id, err);
                        }
                        else if (numberAffected !== 1) {
                            console.error("Invalid number of Facebook posts set as not merged (" + post.foreign_id + ", " + numberAffected + "): " + rawResponse);
                        }
                        // We handle error independently
                        cb(null);
                    });
                }
            }, processed_ids_list);
        }, function (err) {
            process.exit(0);
        });
    });
}

{'fraction_non_alphanumeric': 0.12023233301064859, 'fraction_numerical': 0.0011616650532429815, 'mean_word_length': 1.465871121718377, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2074000', 'n_tokens_mistral': 1405, 'n_tokens_neox': 1300, 'n_words': 311}