<commit_before>var mintcsv =function(number,name) {	

var csv = require('csv');
var fs = require('fs');
var uuid = require('node-uuid');

// var entry =uuid.v1;

console.log(number,name);
// console.log(entry);


var Mds=function MDSToken(account_id,device_id){
	return account_id+ device_id;
}



var oneL=function oneLine(){
	
 return	uuid()+","+uuid()+","+Mds(uuid(),uuid());
	
}

for (var i=0;i < number;++i ){
	
 console.log(oneL());
	
}

return "my firt node moudle :)"
}

module.exports = mintcsv;<commit_msg>no node stream use writefile for more than 400K times write.<commit_after>var number = process.argv[2];
var name = process.argv[3];

console.log(number, name);
var path = require('path');
var csv = require('csv');
var fs = require('fs');
var uuid = require('node-uuid');
var fs = require('fs');
//var Stream = require("stream");
//var wstream = fs.createWriteStream(path.join(__dirname, name));
//var stream = new Stream();
var stream = [];

//stream.readable = true;
//stream.pipe(wstream,{end:false});

//decMDS();

var Mds = function MDSToken(account_id, device_id) {
//    return account_id + device_id;
    var mdstoken = '<KEY>';
    return mdstoken;
};
//

function showDrain() {
    console.log("write drain !!!");
}

var i = 0;
for (i = 1; i <= number; ++i) {
    var accound_id = uuid();
    var random = Math.floor((Math.random() * 6) + 1);
    for(var j=1; j<=random;j++) {

        var device_id = uuid();
        var MdsToken = Mds(accound_id, device_id);
        var displayName = i.toString()+'|'+ j.toString();
        var oneLine = accound_id + "," + device_id + "," + displayName + "," + MdsToken + "\n";
        stream.push((oneLine));
    }
}

if (i >= number) {
    stream.join('\n');
    fs.writeFile(name,stream.join(''));
    console.log("done close");
}


//writeOneMillionTimes(wstream,decMDS(),encoding,callback);


function decMDS() {

    var mdstoken = '<KEY>';
//    console.log("mdstoken lenth ", mdstoken.length);
//    var base64token = decode_base64(mdstoken);
//    console.log(base64token, base64token.length);
    return mdstoken;

}

function decode_base64(s) {
    var e = {}, i, k, v = [], r = '', w = String.fromCharCode;
    var n = [
        [65, 91],
        [97, 123],
        [48, 58],
        [43, 44],
        [47, 48]
    ];

    for (z in n) {
        for (i = n[z][0]; i < n[z][1]; i++) {
            v.push(w(i));
        }
    }
    for (i = 0; i < 64; i++) {
        e[v[i]] = i;
    }

    for (i = 0; i < s.length; i += 72) {
        var b = 0, c, x, l = 0, o = s.substring(i, i + 72);
        for (x = 0; x < o.length; x++) {
            c = e[o.charAt(x)];
            b = (b << 6) + c;
            l += 6;
            while (l >= 8) {
                r += w((b >>> (l -= 8)) % 256);
            }
        }
    }
    return r;
}

function writeOneMillionTimes(writer, data, encoding, callback) {
    var loops = number;
    writecsv();
    function writecsv() {
        var ok = true;
        do {
            loops -= 1;
            if (loops === 0) {
                // last time!
                writer.write(data, encoding, callback);
            } else {
                // see if we should continue, or wait
                // don't pass the callback, because we're not done yet.
                ok = writer.write(data, encoding);
            }
        } while (i > 0 && ok);
        if (i > 0) {
            // write some more once it drains
            writer.once('drain', writecsv);
        }
    }
}

//}

//module.exports = mintcsv;
{'fraction_non_alphanumeric': 0.18261857426867367, 'fraction_numerical': 0.019028685032661176, 'mean_word_length': 2.17011701170117, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3888616', 'n_tokens_mistral': 1291, 'n_tokens_neox': 1217, 'n_words': 313}