<commit_before>const Promise = require('bluebird');
const _ = require('lodash');
const moment = require('moment');
const debug = require('debug')('lib:rss');
const nconf= require('nconf');
const RSS = require('rss');
const path = require('path');
const fs = Promise.promisifyAll(require('fs'));

const utils = require('../lib/utils');
const echoes = require('../lib/echoes');
const mongo = require('../lib/mongo');

const fbtrexRSSdescription = "This newsfeed is generated by the distributed observation of Facebook posts, collected with https://facebook.tracking.exposed browser extension; The posts are processed with a technical proceduce called semantic analysis, it extract the core meanings of the post linkable to existing wikipedia pages";
const fbtrexRSSproblem = "We can't provide a newsfeed on the information you requested. This is, normally, due because you look for a keyword which has not been seen recently. We permit to generate RSS only about voices which are part of wikipedia because this ensure we do not enable any kind of stalking. (editing wikipedia would not work). You can really use only label which are meaningful encyclopedic voices";

function buildXML(lang, label, enriched) {

    if(!_.size(enriched))
        return {
            lang,
            label,
            content: produceError(lang, label, `Not found "${label}" in the last 7 days`)
        };

    const language = _.first(_.first(enriched).labels).lang;
    const title = `fbTREX ⏩ ${lang} ∕ ${label}`;
    const composed = new RSS({
        title,
        description: fbtrexRSSdescription,
        generator: `fbtrexRSS, algorithm version 0`,
        feed_url: `https://facebook.tracking.exposed/feeds/0/${lang}/${encodeURIComponent(label)}`,
        ttl: 60
    });

    /* > _.keys(_.first(enriched)) 
       > [ '_id', 'summary', 'labels' ]              */
    _.each(enriched, function(e) {
        try {
            composed.item(appendItem(e))
        } catch(error) {
            debug("--- Impossible to add an item: %s", error.message);
        }
    });
    debug("Generated RSS (%d bytes, from %d posts) for [%s] %s",
        _.size(composed.xml()), _.size(enriched), lang, label);

    return {
        lang,
        label,
        content: composed.xml()
    };
}

function appendItem(e) {
    /* every entry is an enriched post,
     * this function creates an RSS post (a.k.a item for RSS format) */
    const summary = _.first(e.summary);
    const labels = _.first(e.labels);

    const printableTime = summary.publicationTime ? summary.publicationTime : moment();
    const vdiff = summary.publicationTime ? 
        "in " + moment.duration(
            moment(summary.impressionTime) -
            moment(summary.publicationTime) 
        ).humanize() : null;

    const images = (summary.images && summary.images.count) ? `${summary.images.count} img` : null;
    const opti = _.reduce({
        'LIKE'  :   '👍',
        'LOVE'  :   '❤',
        'ANGRY' :   '💢',
        'WOW'   :   '💡',
        'SAD'   :   '😢',
        'HAHA'  :   '😄' }, function(memo, emoji, rname) {
            var amount = _.get(summary, rname);
            if(_.isUndefined(amount))
                return memo;
            memo.push(`${emoji} ${amount}`);
            return memo;
    }, [ images, vdiff ]);

    const infot = _.compact(opti).join('⍘');
    const title = `[${summary.source} 🖉 ${summary.textsize}] ${infot}`;
    const tldr = _.reduce(summary.texts, function(memo, o) {
        memo += `→ ${o}\n`;
        return memo;
    }, "");
    const link = (summary.opengraph && summary.opengraph.isValid) ? `➭ ${summary.opengraph.link}\n` : "";
    const formattedText = `\n${tldr}\n\n${link}`;
    const concepts = `\n⇉ ${_.uniq(labels.l).join(', ')}\n`;

    let permaLink = null;
    if(_.isNull(summary.permaLink)) {
        permaLink = summary.postId ? `https://facebook.com/${summary.postId}` : "https://facebook.tracking.exposed/fbtrexRSS#problem";
    } else if(!_.startsWith(summary.permaLink, '/')) {
        debug("permaLink do not start with '/': %s", summary.permaLink);
        permaLink = summary.permaLink;
    } else {
        permaLink = summary.permaLink ? `https://facebook.com${summary.permaLink}` : postIdLink;
    }

    /* this is the item format expected by RSS */
    return {
        title,
        description: `${formattedText}${concepts}`,
        url: permaLink,
        guid: summary.semanticId,
        date: printableTime,<commit_msg>fixed a \r\n thing in RSS<commit_after>const Promise = require('bluebird');
const _ = require('lodash');
const moment = require('moment');
const debug = require('debug')('lib:rss');
const nconf= require('nconf');
const RSS = require('rss');
const path = require('path');
const fs = Promise.promisifyAll(require('fs'));

const utils = require('../lib/utils');
const echoes = require('../lib/echoes');
const mongo = require('../lib/mongo');

const fbtrexRSSdescription = "This newsfeed is generated by the distributed observation of Facebook posts, collected with https://facebook.tracking.exposed browser extension; The posts are processed with a technical proceduce called semantic analysis, it extract the core meanings of the post linkable to existing wikipedia pages";
const fbtrexRSSproblem = "We can't provide a newsfeed on the information you requested. This is, normally, due because you look for a keyword which has not been seen recently. We permit to generate RSS only about voices which are part of wikipedia because this ensure we do not enable any kind of stalking. (editing wikipedia would not work). You can really use only label which are meaningful encyclopedic voices";

function buildXML(lang, label, enriched) {

    if(!_.size(enriched))
        return {
            lang,
            label,
            content: produceError(lang, label, `Not found "${label}" in the last 7 days`)
        };

    const language = _.first(_.first(enriched).labels).lang;
    const title = `fbTREX ⏩ ${lang} ∕ ${label}`;
    const composed = new RSS({
        title,
        description: fbtrexRSSdescription,
        generator: `fbtrexRSS, algorithm version 0`,
        feed_url: `https://facebook.tracking.exposed/feeds/0/${lang}/${encodeURIComponent(label)}`,
        ttl: 60
    });

    /* > _.keys(_.first(enriched)) 
       > [ '_id', 'summary', 'labels' ]              */
    _.each(enriched, function(e) {
        try {
            composed.item(appendItem(e))
        } catch(error) {
            debug("--- Impossible to add an item: %s", error.message);
        }
    });
    debug("Generated RSS (%d bytes, from %d posts) for [%s] %s",
        _.size(composed.xml()), _.size(enriched), lang, label);

    return {
        lang,
        label,
        content: composed.xml()
    };
}

function appendItem(e) {
    /* every entry is an enriched post,
     * this function creates an RSS post (a.k.a item for RSS format) */
    const summary = _.first(e.summary);
    const labels = _.first(e.labels);

    const printableTime = summary.publicationTime ? summary.publicationTime : moment();
    const vdiff = summary.publicationTime ? 
        "in " + moment.duration(
            moment(summary.impressionTime) -
            moment(summary.publicationTime) 
        ).humanize() : null;

    const images = (summary.images && summary.images.count) ? `${summary.images.count} img` : null;
    const opti = _.reduce({
        'LIKE'  :   '👍',
        'LOVE'  :   '❤',
        'ANGRY' :   '💢',
        'WOW'   :   '💡',
        'SAD'   :   '😢',
        'HAHA'  :   '😄' }, function(memo, emoji, rname) {
            var amount = _.get(summary, rname);
            if(_.isUndefined(amount))
                return memo;
            memo.push(`${emoji} ${amount}`);
            return memo;
    }, [ images, vdiff ]);

    const infot = _.compact(opti).join('⍘');
    const title = `[${summary.source} 🖉 ${summary.textsize}] ${infot}`;
    const tldr = _.reduce(summary.texts, function(memo, o) {
        memo += `→ ${o}\r\n`;
        return memo;
    }, "");
    const link = (summary.opengraph && summary.opengraph.isValid) ? `➭ ${summary.opengraph.link}\r\n` : "";
    const formattedText = `\r\n${tldr}\r\n\r\n${link}`;
    const concepts = `\r\n⇉ ${_.uniq(labels.l).join(', ')}\r\n`;

    let permaLink = null;
    if(_.isNull(summary.permaLink)) {
        permaLink = summary.postId ? `https://facebook.com/${summary.postId}` : "https://facebook.tracking.exposed/fbtrexRSS#problem";
    } else if(!_.startsWith(summary.permaLink, '/')) {
        debug("permaLink do not start with '/': %s", summary.permaLink);
        permaLink = summary.permaLink;
    } else {
        permaLink = summary.permaLink ? `https://facebook.com${summary.permaLink}` : postIdLink;
    }

    /* this is the item format expected by RSS */
    return {
        title,
        description: `${formattedText}${concepts}`,
        url: permaLink,
        guid: summary.semanticId,
        date: printableTime,
{'fraction_non_alphanumeric': 0.14816053511705685, 'fraction_numerical': 0.0011148272017837235, 'mean_word_length': 2.7177786987152923, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '41829', 'n_tokens_mistral': 2808, 'n_tokens_neox': 2738, 'n_words': 754}