<commit_before>"use strict";

var db = require("../lib/database");
var config = require("../../config");
var hostnameToCollection = require("../../lib/hostname-to-collection");
var JSONStream = require("JSONStream");

var Batch = function () {
    this.items = [];
};

Batch.prototype.push = function (item) {
    this.items.push(item);
};

Batch.prototype.statistics  = function () {
    var items = this.items;
    if (items.length == 0) return {};

    var keys = Object.keys(items[0]);

    return items.reduce(function (p, c, i) {
        keys.forEach(function (key) {
            var value = c[key];
            if (value < p.min[key] || p.min[key] == undefined) p.min[key] = value;
            if (value > p.max[key] || p.max[key] == undefined) p.max[key] = value;
            p.mean[key] = (p.mean[key] || 0) + value;

            if (i == items.length - 1) {
                p.mean[key] = p.mean[key] / items.length;
            }
        });

        return p;
    }, {
        min: {},
        max: {},
        mean: {},
        count: items.length
    });
};

Batch.prototype.reset = function () {
    this.items = [];
};

var round = function (date, timespan) {
    return parseInt(date.getTime() / timespan, 10);
};

config.domains.forEach(function (domain) {
    var collection = hostnameToCollection(domain);
    db.bind(collection, {
        statistics: function (timespan, from, to) {
            var s = JSONStream.stringify();
            this.open(function (err, collection) {
                if (err) {
                    console.dir(err);
                    return s.end();
                }

                var timeStamp = 0;
                var batch = new Batch;

                var dataStream = collection.find({}, {
                    _id: 0,
                    "performance.timing": 1,
                    "createdAt" : 1
                }).sort({createdAt: -1}).stream();

                dataStream.on("data", function (data) {
                    var newTimeStamp = round(data.createdAt, timespan);

                    if (timeStamp == 0) {
                        timeStamp = newTimeStamp;
                    }

                    if (timeStamp > newTimeStamp) {
                        timeStamp = newTimeStamp;
                        var stats = batch.statistics();
                        stats.timestamp = data.createdAt.getTime();
                        s.write(stats);
                        batch.reset();
                    }

                    batch.push(data.performance.timing);
                });

                dataStream.on("end", function (data) {
                    if (data) batch.push(data.performance.timing);
                    s.write(batch.statistics());
                    s.end();
                });

            });
            return s;
        }<commit_msg>use a streaming versions of stats<commit_after>"use strict";

var db = require("../lib/database");
var config = require("../../config");
var hostnameToCollection = require("../../lib/hostname-to-collection");
var JSONStream = require("JSONStream");

var Batch = function () {
    this.reset();
};

Batch.prototype.add = function (item) {
    var s = this.stats;
    var keys = this.keys(item);

    keys.forEach(function (key) {
        var value = item[key];
        if (value < s.min[key] || s.min[key] == undefined) s.min[key] = value;
        if (value > s.max[key] || s.max[key] == undefined) s.max[key] = value;
        s.mean[key] = (((s.mean[key] || 0) * s.count) + value) / (s.count + 1);
    });

    s.count += 1;
};

Batch.prototype.keys = function (item) {
   return this.timingKeys || Object.keys(item);
};

Batch.prototype.statistics  = function () {
    return this.stats;
};

Batch.prototype.reset = function () {
    this.stats = {
        min: {},
        max: {},
        mean: {},
        count: 0
    };
};

var round = function (date, timespan) {
    return parseInt(date.getTime() / timespan, 10);
};

config.domains.forEach(function (domain) {
    var collection = hostnameToCollection(domain);
    db.bind(collection, {
        statistics: function (timespan, from, to) {
            var s = JSONStream.stringify();
            this.open(function (err, collection) {
                if (err) {
                    console.dir(err);
                    return s.end();
                }

                var timeStamp = 0;
                var batch = new Batch;

                var dataStream = collection.find({}, {
                    _id: 0,
                    "performance.timing": 1,
                    "createdAt" : 1
                }).sort({createdAt: -1}).stream();

                dataStream.on("data", function (data) {
                    var newTimeStamp = round(data.createdAt, timespan);

                    if (timeStamp == 0) {
                        timeStamp = newTimeStamp;
                    }

                    if (timeStamp > newTimeStamp) {
                        timeStamp = newTimeStamp;
                        var stats = batch.statistics();
                        stats.timestamp = data.createdAt.getTime();
                        s.write(stats);
                        batch.reset();
                    }

                    batch.add(data.performance.timing);
                });

                dataStream.on("end", function (data) {
                    if (data) batch.push(data.performance.timing);
                    s.write(batch.statistics());
                    s.end();
                });

            });
            return s;
        }
{'fraction_non_alphanumeric': 0.1422661870503597, 'fraction_numerical': 0.004316546762589928, 'mean_word_length': 1.4209838920330866, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7524863', 'n_tokens_mistral': 1612, 'n_tokens_neox': 1506, 'n_words': 325}