<commit_before>console.log('Loading lexique sheet, please wait. It could take a while the first time you use this generator.');
var _                  = require('lodash');
var fs                 = require('fs');
var interestingColumns = [
    {
        name: 'spelling',
        column: 0
    }, {
        name: 'grammar',
        column: 3
    }
];

// If file lexique.json exists, we will use it, else we build it for the first time.
try {
    fs.accessSync(__dirname + '/../assets/lexique.json', fs.F_OK);
    var jsonfile = require('jsonfile');
    var lexique = jsonfile.readFileSync(__dirname + '/../assets/lexique.json');
} catch (e) {
    // Importing xlsb
    var xlsx               = require('node-xlsx');
    var xlsxToJson         = require('./xlsxToJson');
    var lexiqueName        = 'lexique380.xlsb';
    var lexiqueXls         = xlsx.parse(__dirname + '/../assets/' + lexiqueName);
    var sheet              = lexiqueXls[0].data;

    // Construct the usable data, you can modify the method if you want another format
    var lexique = construct(sheet);
    xlsxToJson(lexique); // Save JSON file
}

// Possible sentence construction
var sentenceAvailableConstructions = [
    'PRO:per VER ART:ind NOM ADV AUX ADJ',
    'PRO:per VER ART:def NOM ADV AUX ADJ',
    'PRE ART:def VER ADV ADJ:ind NOM',
    'PRO:per VER PRO:pos NOM',
];

module.exports = {
    generate: function(options) {
        return makeWholeArticle(lexique, options);
    },
    getLexique: function() {
        return lexique;
    }
};

/**
 * Constructs a usable array of data
 * @param  {array} data Raw format data
 * @return {array}      Without extra data
 */
function construct(data) {
    data = data.slice(1); // Remove headers
    var words = [];
    // Retrieves all words and categorizes them
    data.forEach(function(row, key) {
        var word = {};
        interestingColumns.forEach(function(interestingColumn, key) {
            word[interestingColumn.name] = row[interestingColumn.column];
        });
        words.push(word);
    });

    // Reduces the array
    return _.reduce(words, function(l, n) {
        if (i === 0) {
            l = {};
            i++;
        }
        if (!_.isArray(l[n[interestingColumns[1].name]])) {
            l[n[interestingColumns[1].name]] = [];
        }
        l[n[interestingColumns[1].name]].push(n[interestingColumns[0].name]);
        return l;
    });
}

/**
 * Generates the whole article
 * @param  {array}  data    Input data
 * @param  {object} options The options
 * @return {object}         The full article
 */
var makeWholeArticle = function(data, options) {
    // Save the options<commit_msg>Fix i issue<commit_after>console.log('Loading lexique sheet, please wait. It could take a while the first time you use this generator.');
var _                  = require('lodash');
var fs                 = require('fs');
var interestingColumns = [
    {
        name: 'spelling',
        column: 0
    }, {
        name: 'grammar',
        column: 3
    }
];

// If file lexique.json exists, we will use it, else we build it for the first time.
try {
    fs.accessSync(__dirname + '/../assets/lexique.json', fs.F_OK);
    var jsonfile = require('jsonfile');
    var lexique = jsonfile.readFileSync(__dirname + '/../assets/lexique.json');
} catch (e) {
    // Importing xlsb
    var xlsx               = require('node-xlsx');
    var xlsxToJson         = require('./xlsxToJson');
    var lexiqueName        = 'lexique380.xlsb';
    var lexiqueXls         = xlsx.parse(__dirname + '/../assets/' + lexiqueName);
    var sheet              = lexiqueXls[0].data;

    // Construct the usable data, you can modify the method if you want another format
    var lexique = construct(sheet);
    xlsxToJson(lexique); // Save JSON file
}

// Possible sentence construction
var sentenceAvailableConstructions = [
    'PRO:per VER ART:ind NOM ADV AUX ADJ',
    'PRO:per VER ART:def NOM ADV AUX ADJ',
    'PRE ART:def VER ADV ADJ:ind NOM',
    'PRO:per VER PRO:pos NOM',
];

module.exports = {
    generate: function(options) {
        return makeWholeArticle(lexique, options);
    },
    getLexique: function() {
        return lexique;
    }
};

/**
 * Constructs a usable array of data
 * @param  {array} data Raw format data
 * @return {array}      Without extra data
 */
function construct(data) {
    data = data.slice(1); // Remove headers
    var words = [];
    // Retrieves all words and categorizes them
    data.forEach(function(row, key) {
        var word = {};
        interestingColumns.forEach(function(interestingColumn, key) {
            word[interestingColumn.name] = row[interestingColumn.column];
        });
        words.push(word);
    });

    // Reduces the array
    var i = 0;
    return _.reduce(words, function(l, n) {
        if (i === 0) {
            l = {};
            i++;
        }
        if (!_.isArray(l[n[interestingColumns[1].name]])) {
            l[n[interestingColumns[1].name]] = [];
        }
        l[n[interestingColumns[1].name]].push(n[interestingColumns[0].name]);
        return l;
    });
}

/**
 * Generates the whole article
 * @param  {array}  data    Input data
 * @param  {object} options The options
 * @return {object}         The full article
 */
var makeWholeArticle = function(data, options) {
    // Save the options
{'fraction_non_alphanumeric': 0.13130744700806604, 'fraction_numerical': 0.004689551678859501, 'mean_word_length': 2.33458411507192, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4976689', 'n_tokens_mistral': 1667, 'n_tokens_neox': 1599, 'n_words': 452}