<commit_before>const _ = require('lodash');
const moment = require('moment');
const Promise = require('bluebird');
const debug = require('debug')('routes:semantics');
const nconf = require('nconf');

const mongo = require('../lib/mongo');
const params = require('../lib/params');
const semantic = require('../lib/semantic');
const various = require('../lib/various');

const LanguageError = {
    error: true,
    message: 'missing language',
    supported: semantic.langMap,
    reminder: 'the list of supported language comes from a mongodb.distinct call, should be updated'
};

function validLanguage(propl) {
    return (_.size(propl) == 2 && !_.isUndefined(_.get(semantic.langMap, propl)));
}

function labels(req) {
    const { amount, skip } = params.optionParsing(req.params.paging, 100);
    debug("labels request (lang: %s), amount %d skip %d",
        req.params.lang, amount, skip);

    if(!validLanguage(req.params.lang))
        return { json: LanguageError };

    return mongo
        .readLimit(nconf.get('schema').labels, { lang: req.params.lang }, { when: -1 }, amount, skip)
        .then(function(data) {
            debug("retrived %d objects, with amount %d skip %d", _.size(data), amount, skip);
            return { json: data };
        })
        .catch(function(e) {
            debug("data (error): %s", e);
            return { 'text': `error: ${e}` };
        });
};

function semantics(req) {
    const { amount, skip } = params.optionParsing(req.params.paging, 100);
    debug("semantic request (lang: %s), amount %d skip %d",
        req.params.lang, amount, skip);

    if(!validLanguage(req.params.lang))
        return { json: LanguageError };

    return mongo
        .readLimit(nconf.get('schema').semantics, { lang: req.params.lang }, { when: -1 }, amount, skip)
        .then(function(data) {
            debug("retrived %d objects, with amount %d skip %d", _.size(data), amount, skip);
            return { json: data };
        })
        .catch(function(e) {
            debug("data (error): %s", e);
            return { 'text': `error: ${e}` };
        });
};

function redactEnriched(enrich) {
    _.unset(enrich, '_id');
    let s = _.first(enrich.summary);
    _.unset(s, '_id');
    _.unset(s, 'id');
    _.unset(s, 'user');
    _.unset(s, 'timeline');
    _.unset(s, 'impressionOrder');
    _.unset(s, 'impressionTime');
    enrich.summary = s;
    // XXX maybe can be redacted the condition of the ❌ below
    return enrich;
}

function enrich(req) {
    const DEFAULTAMOUNT = 13;
    const backintime = moment().subtract(2, 'd').toISOString();
    const { amount, skip } = params.optionParsing(req.params.paging, DEFAULTAMOUNT);
    debug("enrich request (lang: %s), amount %d skip %d",
        req.params.lang, amount, skip);

    if(!validLanguage(req.params.lang))
        return { json: LanguageError };

    return mongo
        .aggregate(nconf.get('schema').labels, [
            { $sort: { when: -1 }},
            { $match: { lang: req.params.lang, when: { "$lt": new Date(backintime) }} },
            { $skip: skip },
            { $limit: amount },
            { $lookup: { from: 'summary', localField: 'semanticId', foreignField: 'semanticId', as: 'summary' }}
        ])
        .map(redactEnriched)
        .then(function(enriched) {
            debug("return enrich, %d objects (%j)", _.size(enriched), _.map(enriched, function(e) {
                if(!e.summary || !e.summary.source || !e.summary.nature)
                    return "❌";
                return [ e.summary.source, e.summary.nature ];
            }));
            return { json: enriched };
        });
};

function noogle(req) {
    const DEFAULTAMOUNT = 13;
    const { amount, skip } = params.optionParsing(req.params.paging, DEFAULTAMOUNT);
    debug("noogle request (lang: %s, label: %s), amount %d skip %d",
        req.params.lang, req.params.label, amount, skip);

    if(!validLanguage(req.params.lang))
        return { json: LanguageError };

    return mongo
        .aggregate(nconf.get('schema').semantics, [
            { $sort: { when: -1 }},
            { $match: { lang: req.params.lang, label: req.params.label }},
            { $group: { _id: "$semanticId" }},
            { $skip: skip },<commit_msg>fixed a bug in noogle<commit_after>const _ = require('lodash');
const moment = require('moment');
const Promise = require('bluebird');
const debug = require('debug')('routes:semantics');
const nconf = require('nconf');

const mongo = require('../lib/mongo');
const params = require('../lib/params');
const semantic = require('../lib/semantic');
const various = require('../lib/various');

const LanguageError = {
    error: true,
    message: 'missing language',
    supported: semantic.langMap,
    reminder: 'the list of supported language comes from a mongodb.distinct call, should be updated'
};

function validLanguage(propl) {
    return (_.size(propl) == 2 && !_.isUndefined(_.get(semantic.langMap, propl)));
}

function labels(req) {
    const { amount, skip } = params.optionParsing(req.params.paging, 100);
    debug("labels request (lang: %s), amount %d skip %d",
        req.params.lang, amount, skip);

    if(!validLanguage(req.params.lang))
        return { json: LanguageError };

    return mongo
        .readLimit(nconf.get('schema').labels, { lang: req.params.lang }, { when: -1 }, amount, skip)
        .then(function(data) {
            debug("retrived %d objects, with amount %d skip %d", _.size(data), amount, skip);
            return { json: data };
        })
        .catch(function(e) {
            debug("data (error): %s", e);
            return { 'text': `error: ${e}` };
        });
};

function semantics(req) {
    const { amount, skip } = params.optionParsing(req.params.paging, 100);
    debug("semantic request (lang: %s), amount %d skip %d",
        req.params.lang, amount, skip);

    if(!validLanguage(req.params.lang))
        return { json: LanguageError };

    return mongo
        .readLimit(nconf.get('schema').semantics, { lang: req.params.lang }, { when: -1 }, amount, skip)
        .then(function(data) {
            debug("retrived %d objects, with amount %d skip %d", _.size(data), amount, skip);
            return { json: data };
        })
        .catch(function(e) {
            debug("data (error): %s", e);
            return { 'text': `error: ${e}` };
        });
};

function redactEnriched(enrich) {
    _.unset(enrich, '_id');
    let s = _.first(enrich.summary);
    _.unset(s, '_id');
    _.unset(s, 'id');
    _.unset(s, 'user');
    _.unset(s, 'timeline');
    _.unset(s, 'impressionOrder');
    _.unset(s, 'impressionTime');
    enrich.summary = s;
    // XXX maybe can be redacted the condition of the ❌ below
    return enrich;
}

function enrich(req) {
    const DEFAULTAMOUNT = 13;
    const backintime = moment().subtract(2, 'd').toISOString();
    const { amount, skip } = params.optionParsing(req.params.paging, DEFAULTAMOUNT);
    debug("enrich request (lang: %s), amount %d skip %d",
        req.params.lang, amount, skip);

    if(!validLanguage(req.params.lang))
        return { json: LanguageError };

    return mongo
        .aggregate(nconf.get('schema').labels, [
            { $sort: { when: -1 }},
            { $match: { lang: req.params.lang, when: { "$lt": new Date(backintime) }} },
            { $skip: skip },
            { $limit: amount },
            { $lookup: { from: 'summary', localField: 'semanticId', foreignField: 'semanticId', as: 'summary' }}
        ])
        .map(redactEnriched)
        .then(function(enriched) {
            debug("return enrich, %d objects (%j)", _.size(enriched), _.map(enriched, function(e) {
                if(!e.summary || !e.summary.source || !e.summary.nature)
                    return "❌";
                return [ e.summary.source, e.summary.nature ];
            }));
            return { json: enriched };
        });
};

function noogle(req) {
    const DEFAULTAMOUNT = 13;
    const { amount, skip } = params.optionParsing(req.params.paging, DEFAULTAMOUNT);
    debug("noogle request (lang: %s, title: %s), amount %d skip %d",
        req.params.lang, req.params.label, amount, skip);

    if(!validLanguage(req.params.lang))
        return { json: LanguageError };

    return mongo
        .aggregate(nconf.get('schema').semantics, [
            { $sort: { when: -1 }},
            { $match: { lang: req.params.lang, title: req.params.label }},
            { $group: { _id: "$semanticId" }},
            { $skip: skip },
{'fraction_non_alphanumeric': 0.169638667292351, 'fraction_numerical': 0.003754106053496011, 'mean_word_length': 2.5894736842105264, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3175610', 'n_tokens_mistral': 2734, 'n_tokens_neox': 2651, 'n_words': 644}