<commit_before>/*
 * mood
 * https://github.com/diesire/mood
 *
 * Copyright (c) 2013 <NAME>
 * Licensed under the MIT license.
 */

'use strict';

var natural = require('natural'),
    LanguageDetect = require('languagedetect');


var tools = {
    tokenizer: function(text) {
        return new natural.WordTokenizer().tokenize(text);
    },
    stemmer: function(text) {
        natural.PorterStemmer.attach();
        return text.tokenizeAndStem();
    },
    language: function(text) {
        return new LanguageDetect().detect(text);
    }
};

// MongoDB backend
(function(exports) {
    var mongoose = require('mongoose');
    var config = require('./config');
    console.log(config);
    mongoose.connect(config.creds.mongoose_auth_devel);
    var db = mongoose.connection;
    db.on('error', console.error.bind(console, 'connection error:'));
    db.once('open', function callback() {
        console.log('ok---------------');
    });
    var Schema = mongoose.Schema;

    // Create a schema for our data
    var SeedSchema = new Schema({
        word_en: String,
        stemm_en: String,
        word_es: String,
        stemm_es: String,
        mean: Number,
        std_deviation: Number
    });

    // Use the schema to register a model with MongoDb
    mongoose.model('Seed', SeedSchema);
    var Seed = mongoose.model('Seed');

    // 
    exports.get = function(condition, callback) {
        Seed.find(condition, '-_id', callback);
    };

    exports.save = function(callback) {
        var s = new Seed();
        s.word_en = 'qwerty';
        s.stemm_en = 'qwe';
        s.word_es = 'pollo';
        s.stemm_es = 'po';
        s.mean = 5;
        s.std_deviation = 1.2;

        s.save(callback);
    };

    exports.getAll = function(callback) {
        Seed.find({
            word_es: 'abadía'
        }).select('mean').limit(1).exec(callback);
    };
})(exports);


exports.rate = function(text, callback) {

    //empty text -> Error
    if (text === undefined || text === '') {
        throw new Error('Invalid argument');
    }
    
    //tweet -> stemms -> 
    var stemms = tools.stemmer(text);
    stemms.map(function(stemm){
        get({stemm_es: stemm}, callback);
    });

    return text.length;
};


<commit_msg>Add batchStemmer<commit_after>/*
 * mood
 * https://github.com/diesire/mood
 *
 * Copyright (c) 2013 <NAME>
 * Licensed under the MIT license.
 */

'use strict';

var natural = require('natural'),
    LanguageDetect = require('languagedetect');


var tools = {
    tokenizer: function(text) {
        return new natural.WordTokenizer().tokenize(text);
    },
    stemmer: function(text) {
        natural.PorterStemmer.attach();
        return text.tokenizeAndStem();
    },
    language: function(text) {
        return new LanguageDetect().detect(text);
    }
};

// MongoDB backend
(function(exports) {
    var mongoose = require('mongoose');
    var config = require('./config');
    console.log(config);
    mongoose.connect(config.creds.mongoose_auth_devel);
    var db = mongoose.connection;
    db.on('error', console.error.bind(console, 'connection error:'));
    db.once('open', function callback() {
        console.log('ok---------------');
    });
    var Schema = mongoose.Schema;

    // Create a schema for our data
    var SeedSchema = new Schema({
        word_en: String,
        stemm_en: String,
        word_es: String,
        stemm_es: String,
        mean: Number,
        std_deviation: Number
    });

    // Use the schema to register a model with MongoDb
    mongoose.model('Seed', SeedSchema);
    var Seed = mongoose.model('Seed');

    // 
    exports.get = function(condition, callback) {
        Seed.find(condition, '-_id', callback);
    };

    exports.save = function(callback) {
        var s = new Seed();
        s.word_en = 'qwerty';
        s.stemm_en = 'qwe';
        s.word_es = 'pollo';
        s.stemm_es = 'po';
        s.mean = 5;
        s.std_deviation = 1.2;

        s.save(callback);
    };

    exports.getAll = function(callback) {
        Seed.find({
            word_es: 'abadía'
        }).select('mean').limit(1).exec(callback);
    };
})(exports);


exports.rate = function(text, callback) {
    exports.batchStemmer = function(callback) {
        Seed.find().exec(function(err, seeds) {
            if (err) {
                callback('Error in find', null);
                return;
            }
            seeds.forEach(function(seed) {
                var stemm_es = tools.stemmer(seed.word_es, 'es');
                var stemm_en = tools.stemmer(seed.word_en, 'en');
                seed.stemm_en = stemm_en;
                seed.stemm_es = stemm_es;
                seed.save(function() {
                    console.log('Update seed %s - %s / %s - %s', seed.word_es,
                        stemm_es, seed.word_en, stemm_en);
                });
            });
            callback(null, seeds);
        });
    };
})(exports);

    //empty text -> Error
    if (text === undefined || text === '') {
        throw new Error('Invalid argument');
    }
    
    //tweet -> stemms -> 
    var stemms = tools.stemmer(text);
    stemms.map(function(stemm){
        get({stemm_es: stemm}, callback);
    });

    return text.length;
};



{'fraction_non_alphanumeric': 0.15220700152207, 'fraction_numerical': 0.0030441400304414, 'mean_word_length': 2.1198813056379824, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3651332', 'n_tokens_mistral': 1698, 'n_tokens_neox': 1626, 'n_words': 347}