<commit_before>var fs = require('fs'),
    path = require('path'),
    csv = require('csv-parser'),
    levelup = require('levelup'),
    key = require('./lib/key.js'),
    queue = require('queue-async'),
    rimraf = require('rimraf');

var verbose = false;
if (require.main === module) {
    verbose = true;
    if (process.argv[2] === undefined) {
        return console.log('file argument required \n`node import-csv.js [source csv]`');
    }
    loadTask(process.argv[2], function() { });
}

module.exports = loadTask;

function loadTask(fileLoc, callback) {    
    var task = path.basename(fileLoc).split('.')[0],
        topq = queue();

    // ensure that we create & close the tracking database before loading anything
    topq.defer(function(tqcallback){
        levelup('./ldb/' + task + '-tracking.ldb', {}, function(err, trackingdb){
            if (err) return tqcallback(err);
            trackingdb.close(function(err){                                                
                tqcallback(err);
            });
        });
    });

    topq.awaitAll(function(err, results){
        // @TODO: make fixedlist task-specific instead of global
        
        // perform some tests to see if there's a 'fixed' file for this task
        var fixedq = queue();
        fixedq            
            .defer(function(fqcallback){
                fs.readdir('./fixed', function(err, files){ 
                    if (err) return fqcallback(err); 
                    fqcallback(err, (files.indexOf(task) > -1)); 
                });
            })
            .defer(function(fqcallback){
                fs.stat('./fixed/' + task, function(err, info){ 
                    if (err) return fqcallback(err); 
                    fqcallback(err, (info.size > 0)); 
                });
            });

        // once the tests come back, check if they're all positive. if so, load fixes & tasks.
        fixedq.awaitAll(function(err, results){                        
            if((!err) && results.every(function(x) { return x; })) {<commit_msg>removing @TODO (it was actually already done)<commit_after>var fs = require('fs'),
    path = require('path'),
    csv = require('csv-parser'),
    levelup = require('levelup'),
    key = require('./lib/key.js'),
    queue = require('queue-async'),
    rimraf = require('rimraf');

var verbose = false;
if (require.main === module) {
    verbose = true;
    if (process.argv[2] === undefined) {
        return console.log('file argument required \n`node import-csv.js [source csv]`');
    }
    loadTask(process.argv[2], function() { });
}

module.exports = loadTask;

function loadTask(fileLoc, callback) {    
    var task = path.basename(fileLoc).split('.')[0],
        topq = queue();

    // ensure that we create & close the tracking database before loading anything
    topq.defer(function(tqcallback){
        levelup('./ldb/' + task + '-tracking.ldb', {}, function(err, trackingdb){
            if (err) return tqcallback(err);
            trackingdb.close(function(err){                                                
                tqcallback(err);
            });
        });
    });

    topq.awaitAll(function(err, results){        
        // perform some tests to see if there's a 'fixed' file for this task
        var fixedq = queue();
        fixedq            
            .defer(function(fqcallback){
                fs.readdir('./fixed', function(err, files){ 
                    if (err) return fqcallback(err); 
                    fqcallback(err, (files.indexOf(task) > -1)); 
                });
            })
            .defer(function(fqcallback){
                fs.stat('./fixed/' + task, function(err, info){ 
                    if (err) return fqcallback(err); 
                    fqcallback(err, (info.size > 0)); 
                });
            });

        // once the tests come back, check if they're all positive. if so, load fixes & tasks.
        fixedq.awaitAll(function(err, results){                        
            if((!err) && results.every(function(x) { return x; })) {
{'fraction_non_alphanumeric': 0.14383561643835616, 'fraction_numerical': 0.002446183953033268, 'mean_word_length': 1.7949419002050582, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3772664', 'n_tokens_mistral': 1155, 'n_tokens_neox': 1099, 'n_words': 277}