<commit_before>#!/usr/bin/env nodejs
var _ = require('lodash');
var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));
var debug = require('debug')('national-csv');
var moment = require('moment');
var nconf = require('nconf');

var utils = require('../lib/utils');
var mongo = require('../lib/mongo');
var timutils = require('../lib/timeutils');

var cfgFile = "config/settings.json";
nconf.argv().env().file({ file: cfgFile });

/*
 * when: `ISO date`
 * htmls: `Int`      | number of htmls
 * impressions: `Int`| guess what..
 * accesses: `Int`   | accesses, page views
 * active: `Int`     | timelines, count users + countries
 * new: `Int`        | supporters, `keyTime`
 */

var timeFilter = timutils.doFilter(
        nconf.get('HOURSAFTER'), 
        nconf.get('DAYSAGO'), 
        nconf.get('HOURSAGO'), 30, 'd'
    );
var sourceNation = nconf.get('nation');

if(!sourceNation)
    throw new Error("`sourceNation` as environment variable is required");

var destFile = sourceNation + "-" + moment(timeFilter.start).format("YYYY-MM-DD") +'+30d.csv';

var keys = [
    'id',
    'userPseudo',
    'publicationUTime',
    'impressionTime',
    'impressionOrder',
    'postId',
    'permaLink',
    'timelineId',
    'counter'
];

debug("Executing timewindow: %s and nation %s",
    timutils.prettify(timeFilter), sourceNation);

function getFromHtmls(timeline) {
    return mongo
        .read(nconf.get('schema').htmls, { timelineId: timeline.id, type: 'feed' })
        .reduce(function(memo, e) {
            if(!e || !e.id)
                return memo;

            var metadata = _.pick(e, ['id', 'publicationUTime', 'postId', 'permaLink', 'timelineId' ]);
            memo[e.id] = metadata;
            return memo;
        }, {});
};

function getFromImpressions(timeline) {
    return mongo
        .read(nconf.get('schema').impressions, { timelineId: timeline.id })
        .reduce(function(memo, imp) {
            if(!imp || !imp.htmlId)
                return memo;

            memo[imp.htmlId] = {
                userId: imp.userId,
                impressionOrder: imp.impressionOrder,
                impressionTime: moment(imp.impressionTime).toISOString(),
            };
            return memo;
        }, {});
};

function appendCSV(timeline, counter) {

    return Promise.all([
            getFromHtmls(timeline),
            getFromImpressions(timeline)
    ])
    .then(function(combos) {
        return _.map(combos[0], function(metadata, id) {
            if(!combos[1][id])
                debugger;
            var fix = _.merge(
                    _.pick(metadata, ['permaLink', 'hrefType', 'postId', 'publicationUTime' ]),
                    combos[1][id]
                );
            fix.userPseudo = utils.numId2Words([1, fix.userId]);
            fix.counter = counter;
            return fix;<commit_msg>fixed a bug in fields generation<commit_after>#!/usr/bin/env nodejs
var _ = require('lodash');
var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));
var debug = require('debug')('national-csv');
var moment = require('moment');
var nconf = require('nconf');

var utils = require('../lib/utils');
var mongo = require('../lib/mongo');
var timutils = require('../lib/timeutils');

var cfgFile = "config/settings.json";
nconf.argv().env().file({ file: cfgFile });

/*
 * when: `ISO date`
 * htmls: `Int`      | number of htmls
 * impressions: `Int`| guess what..
 * accesses: `Int`   | accesses, page views
 * active: `Int`     | timelines, count users + countries
 * new: `Int`        | supporters, `keyTime`
 */

var timeFilter = timutils.doFilter(
        nconf.get('HOURSAFTER'), 
        nconf.get('DAYSAGO'), 
        nconf.get('HOURSAGO'), 30, 'd'
    );
var sourceNation = nconf.get('nation');

if(!sourceNation)
    throw new Error("`sourceNation` as environment variable is required");

var destFile = sourceNation + "-" + moment(timeFilter.start).format("YYYY-MM-DD") +'+30d.csv';

var keys = [
    'id',
    'userPseudo',
    'publicationUTime',
    'impressionTime',
    'impressionOrder',
    'postId',
    'permaLink',
    'timelineId',
    'counter'
];

debug("Executing timewindow: %s and nation %s",
    timutils.prettify(timeFilter), sourceNation);

function getFromHtmls(timeline) {
    return mongo
        .read(nconf.get('schema').htmls, { timelineId: timeline.id, type: 'feed' })
        .reduce(function(memo, e) {
            if(!e || !e.id)
                return memo;

            var metadata = _.pick(e, ['id', 'publicationUTime', 'postId', 'permaLink', 'timelineId' ]);
            memo[e.id] = metadata;
            return memo;
        }, {});
};

function getFromImpressions(timeline) {
    return mongo
        .read(nconf.get('schema').impressions, { timelineId: timeline.id })
        .reduce(function(memo, imp) {
            if(!imp || !imp.htmlId)
                return memo;

            memo[imp.htmlId] = {
                userId: imp.userId,
                impressionOrder: imp.impressionOrder,
                impressionTime: moment(imp.impressionTime).toISOString(),
            };
            return memo;
        }, {});
};

function appendCSV(timeline, counter) {

    return Promise.all([
            getFromHtmls(timeline),
            getFromImpressions(timeline)
    ])
    .then(function(combos) {
        return _.map(combos[0], function(metadata, id) {

            if(!combos[1][id])
                throw new Error("impossible!?");

            var fix = _.extend(combos[1][id], metadata);
            fix.userPseudo = utils.numId2Words([1, fix.userId]);
            fix.counter = counter;
            return fix;
{'fraction_non_alphanumeric': 0.15993651913242815, 'fraction_numerical': 0.0031740433785928408, 'mean_word_length': 2.5055624227441284, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2108902', 'n_tokens_mistral': 1834, 'n_tokens_neox': 1793, 'n_words': 369}