<commit_before>
var levelup = require('levelup')
var rimraf = require("rimraf")

if(!module.parent) {

  var dir = '/tmp/map-reduce-sum-test'

  rimraf(dir, function () {

    levelup(dir, {createIfMissing: true}, function (err, db) {

    var l = 10e3, i = 1
    var stream = db.writeStream()
      while(l--)
        stream.write({key: JSON.stringify(i++), value: JSON.stringify(i)})
    })

  })

}

module.exports = sum

function sum(db, callback) {
    rimraf(db, function (err) {
        if (err) {
            return callback(err)
        }

        levelup(db, { createIfMissing: true }, function (err, db) {
            if (err) {
                return callback(err)
            }

            var stream = db.writeStream()

            for(var i = 0; i < 1000; i++) {
                stream.write({
                    key: JSON.stringify(i)
                    , value: JSON.stringify(i)
                })
            }

            stream.end()

            stream.on("close", function () {
                callback(null)
            })
        })
    })
}

<commit_msg>what?<commit_after>
var levelup = require('levelup')
var rimraf  = require('rimraf')

if(!module.parent) {

  var dir = '/tmp/map-reduce-sum-test'

  rimraf(dir, function () {

    levelup(dir, {createIfMissing: true}, function (err, db) {

    var l = 10e3, i = 1
    var stream = db.writeStream()
      while(l--)
        stream.write({key: JSON.stringify(i++), value: JSON.stringify(i)})
    })

  })

}

module.exports = sum

function sum(db, callback) {
    rimraf(db, function (err) {
        if (err) {
            return callback(err)
        }

        levelup(db, { createIfMissing: true }, function (err, db) {
            if (err) {
                return callback(err)
            }

            var stream = db.writeStream()

            for(var i = 0; i < 1000; i++) {
                stream.write({
                    key: JSON.stringify(i)
                    , value: JSON.stringify(i)
                })
            }

            stream.end()

            stream.on("close", function () {
                callback(null)
            })
        })
    })
}


{'fraction_non_alphanumeric': 0.14298935677926886, 'fraction_numerical': 0.008329477093937991, 'mean_word_length': 1.5081206496519721, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1162971', 'n_tokens_mistral': 699, 'n_tokens_neox': 649, 'n_words': 144}