<commit_before><commit_msg>small uncommon script to extract top keywords per language<commit_after>const _ = require('lodash');
const mongo = require('./lib/mongo');

const debug = require('debug')('M');
const nconf = require('nconf');
const cfgFile = "config/content.json";
nconf.argv().env().file({ file: cfgFile })

map = [ { cc: 'dk', name: 'Denmark' },
  { cc: 'ma', name: 'Morocco' },
  { cc: 'it', name: 'Italy' },
  { cc: 'es', name: 'Spain' },
  { cc: 'gr', name: 'Greece' },
  { cc: 'de', name: 'Germany' },
  { cc: 'ch', name: 'Switzerland' },
  { cc: 'fr', name: 'France' },
  { cc: 'gb', name: 'United Kingdom' },
  { cc: 'sk', name: 'Slovakia' },
  { cc: 'cz', name: 'Czech Republic' },
  { cc: 'be', name: 'Belgium' },
  { cc: 'ro', name: 'Romania' },
  { cc: 'hu', name: 'Hungary' },
  { cc: 'pl', name: 'Poland' },
  { cc: 'ie', name: 'Ireland' },
  { cc: 'nl', name: 'Netherlands' },
  { cc: 'pt', name: 'Portugal' },
  { cc: 'se', name: 'Sweden' },
  { cc: 'no', name: 'Norway' },
  { cc: 'lt', name: 'Lithuania' },
  { cc: 'fi', name: 'Finland' },
  { cc: 'at', name: 'Austria' },
  { cc: 'mt', name: 'Malta' },
  { cc: 'hr', name: 'Croatia' },
  { cc: 'cy', name: 'Cyprus' },
  { cc: 'bg', name: 'Bulgaria' },
  { cc: 'me', name: 'Montenegro' },
  { cc: 'lv', name: 'Latvia' },
  { cc: 'ee', name: 'Estonia' } ];


return mongo.distinct('semantics', 'lang')
    .map(function(lang) {
        return mongo
            .readLimit(nconf.get('schema').semantics, { lang: lang}, { when: -1}, 5, 0)
            .then(function(entries) {
                if(!entries || !entries[0])
                    return null;

                var country =  _.find(map, { cc: entries[0].lang} );
                if(!country) {
                    debug("missing country code / lang %s", entries[0].lang);
                    return null;
                }

                return _.reduce(entries, function(memo, e) { 
                    memo.labels.push(e.label);
                    return memo;
                }, {
                    lang,
                    country: country.name,
                    labels: [],
                    total: 100
                });
            });
    }, { concurrency : 1 })
    .then(_.compact)
    .then(function(x) {
        console.log(JSON.stringify(x, undefined, 2));
    });
{'fraction_non_alphanumeric': 0.21204301075268817, 'fraction_numerical': 0.004731182795698925, 'mean_word_length': 2.0930851063829787, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7472997', 'n_tokens_mistral': 856, 'n_tokens_neox': 807, 'n_words': 200}