<commit_before>var pg = require('pg'),
    mongoose = require('mongoose'),
    _ = require('underscore'),
    async = require('async'),
    fs = require('fs'),
    path = require('path'),
    util = require('../server/lib/util'),
    config = require('../server/settings'),
    Group = require('../server/models').Group,
    User = require('../server/models').User;

mongoose.connect('mongodb://localhost/nidarholm');
var client = new pg.Client(config.convert.pg),
    root_path = config.convert.path_files;

var get_permissions = function (groupid) {
    var promise = new mongoose.Promise();

    var permissions = {public: false, groups: [], users: []};
    if (!groupid) {
        permissions.public = true;
        promise.fulfill(permissions);
    } else {
        Group.findOne({old_id: groupid}, function (err, group) {
            if (err) { promise.error(err);}
            permissions.groups.push(group._id);
            promise.fulfill(permissions);
        });
    }
    return promise;
};

client.connect(function(err) {
    if(err) {
        return console.error('could not connect to postgres', err);
    }
    client.query('SELECT * from vault_uploadedfile', function(err, result) {
        if(err) {
            return console.error('error running query', err);
        }

        async.eachLimit(result.rows, 256, function (file, callback) {
            var original_path = path.join(root_path, file.file);
            var exists = fs.existsSync(original_path);
            var basename = path.basename(file.file);

            if (exists) {
                get_permissions(file.group_id).then(function (permissions) {
                    util.upload_file(original_path, file.filename, 'nidarholm.' + file.user_id,
                                     {
                                         tags: util.tagify(file.tags),
                                         permissions: permissions,
                                         do_delete: false,
                                         do_create_duplicates_in_database: false
                                     }, function (err, f) {
                                         if (err && err.message !== "File already exists") {
                                             callback(err);
                                         }
                                         callback();
                                     });
                });
            } else {
                console.error("File not found at " + original_path + ": " + file.filename);
                callback();
            }
        }, function (err) {
            if (err) {
                console.error(err.message);
            } else {
                console.log("Done");
            }
            client.end();<commit_msg>Upload time is important to keep<commit_after>var pg = require('pg'),
    mongoose = require('mongoose'),
    _ = require('underscore'),
    async = require('async'),
    fs = require('fs'),
    path = require('path'),
    util = require('../server/lib/util'),
    config = require('../server/settings'),
    Group = require('../server/models').Group,
    User = require('../server/models').User;

mongoose.connect('mongodb://localhost/nidarholm');
var client = new pg.Client(config.convert.pg),
    root_path = config.convert.path_files;

var get_permissions = function (groupid) {
    var promise = new mongoose.Promise();

    var permissions = {public: false, groups: [], users: []};
    if (!groupid) {
        permissions.public = true;
        promise.fulfill(permissions);
    } else {
        Group.findOne({old_id: groupid}, function (err, group) {
            if (err) { promise.error(err);}
            permissions.groups.push(group._id);
            promise.fulfill(permissions);
        });
    }
    return promise;
};

client.connect(function(err) {
    if(err) {
        return console.error('could not connect to postgres', err);
    }
    client.query('SELECT * from vault_uploadedfile', function(err, result) {
        if(err) {
            return console.error('error running query', err);
        }

        async.eachLimit(result.rows, 256, function (file, callback) {
            var original_path = path.join(root_path, file.file);
            var exists = fs.existsSync(original_path);
            var basename = path.basename(file.file);

            if (exists) {
                get_permissions(file.group_id).then(function (permissions) {
                    util.upload_file(original_path, file.filename, 'nidarholm.' + file.user_id,
                                     {
                                         tags: util.tagify(file.tags),
                                         permissions: permissions,
                                         created: file.uploaded,
                                         do_delete: false,
                                         do_create_duplicates_in_database: false
                                     }, function (err, f) {
                                         if (err && err.message !== "File already exists") {
                                             callback(err);
                                         }
                                         callback();
                                     });
                });
            } else {
                console.error("File not found at " + original_path + ": " + file.filename);
                callback();
            }
        }, function (err) {
            if (err) {
                console.error(err.message);
            } else {
                console.log("Done");
            }
            client.end();
{'fraction_non_alphanumeric': 0.12313961729270022, 'fraction_numerical': 0.0010630758327427356, 'mean_word_length': 1.410333048676345, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6564118', 'n_tokens_mistral': 1392, 'n_tokens_neox': 1371, 'n_words': 280}