<commit_before>#!/usr/bin/env node
var _ = require('lodash');
var Promise = require('bluebird');
var debug = require('debug')('0009-apostrophe');
var moment = require('moment');
var nconf = require('nconf');
var crypto = require('crypto');

var mongo = require('../../lib/mongo');
var utils = require('../../lib/utils');

var cfgFile = "config/content.json";
nconf.argv().env().file({ file: cfgFile });

function convertLabels(offset, slot) {
    let stats = { total: null, done: 0 };
    return mongo
        .readLimit(nconf.get('schema').labels, {}, {}, slot, offset)
        .tap(function(entries) {
            stats.total = _.size(entries);
        })
        .map(function(e) {
            stats.done += 1;
            if(!(stats.done % 300))
                debug("labels: %d/%d, %d%%", stats.done, stats.total, 
                    _.round(((100 / stats.total) * stats.done), 1));
            e.l = _.map(e.l, function(label) {
                return _.replace(label, "'", "’");
            });
            return mongo
                .updateOne(nconf.get('schema').labels, { _id: e._id }, e);
        }, {concurrency: 1})
        .then(_.size);
}

function convertSemantics(offset, slot) {
    let stats = { total: null, done: 0 };
    debug("offset at %d, slot at %d", offset, slot); 
    return mongo
        .readLimit(nconf.get('schema').semantics, {}, {}, slot, offset)
        .tap(function(entries) {
            stats.total = _.size(entries);
        })
        .map(function(e) {
            stats.done += 1;
            if(!(stats.done % 300))
                debug("semantics: %d/%d, %d%%", stats.done, stats.total, 
                    _.round(((100 / stats.total) * stats.done), 1));
            e.title = _.replace(e.title, "'", "’");
            return mongo
                .updateOne(nconf.get('schema').semantics, { _id: e._id }, e);
        }, {concurrency: 1})
        .then(_.size);
}

const offset = _.parseInt(nconf.get('offset'));
if(_.isNaN(offset)) {
    console.log("--offset mandatory");
    process.exit(1);
}


/*
   X:PRIMARY> db.semantics.count()
    1122843
   X:PRIMARY> db.labels.count()
    136162
 */

// labels 0 137
// return convertLabels(offset, 1000)
//     .then(console.log);

// semantics 0 1123
return convertSemantics(offset, 1000)
    .then(console.log);
<commit_msg>fixed convertor for #130<commit_after>#!/usr/bin/env node
var _ = require('lodash');
var Promise = require('bluebird');
var debug = require('debug')('0009-apostrophe');
var moment = require('moment');
var nconf = require('nconf');
var crypto = require('crypto');

var mongo = require('../../lib/mongo');
var utils = require('../../lib/utils');

var cfgFile = "config/content.json";
nconf.argv().env().file({ file: cfgFile });

function convertLabels(offset, slot) {
    let stats = { total: null, done: 0 };
    return mongo
        .readLimit(nconf.get('schema').labels, {}, {}, slot, offset)
        .tap(function(entries) {
            stats.total = _.size(entries);
        })
        .map(function(e) {
            stats.done += 1;
            if(!(stats.done % 300))
                debug("labels: %d/%d, %d%%", stats.done, stats.total, 
                    _.round(((100 / stats.total) * stats.done), 1));
            e.l = _.map(e.l, function(label) {
                return _.replace(label, "'", "’");
            });
            return mongo
                .updateOne(nconf.get('schema').labels, { _id: e._id }, e);
        }, {concurrency: 1})
        .then(_.size);
}

function convertSemantics(offset, slot) {
    let stats = { total: null, done: 0 };
    debug("offset at %d, slot at %d", offset, slot); 
    return mongo
        .readLimit(nconf.get('schema').semantics, {}, {}, slot, offset)
        .tap(function(entries) {
            stats.total = _.size(entries);
        })
        .map(function(e) {
            stats.done += 1;
            if(!(stats.done % 300))
                debug("semantics: %d/%d, %d%%", stats.done, stats.total, 
                    _.round(((100 / stats.total) * stats.done), 1));
            e.title = _.replace(e.title, "'", "’");
            e.label = _.replace(e.label, "'", "’");
            return mongo
                .updateOne(nconf.get('schema').semantics, { _id: e._id }, e);
        }, {concurrency: 1})
        .then(_.size);
}

const offset = _.parseInt(nconf.get('offset'));
if(_.isNaN(offset)) {
    console.log("--offset mandatory");
    process.exit(1);
}


/*
   X:PRIMARY> db.semantics.count()
    1122843
   X:PRIMARY> db.labels.count()
    136162
 */

// labels 0 137
// return convertLabels(offset, 1000)
//     .then(console.log);

// semantics 0 1123
return convertSemantics(offset, 1000)
    .then(console.log);

{'fraction_non_alphanumeric': 0.19088016967126192, 'fraction_numerical': 0.023966065747613997, 'mean_word_length': 2.1907983761840324, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1350963', 'n_tokens_mistral': 1626, 'n_tokens_neox': 1485, 'n_words': 311}