<commit_before>'use strict';

var Dictionary = require('./Dictionary');

var CONSTANTS = require('../Constants');
var path = require('path');
var fs = require('fs');

function CSVDumper(dictFilename) {
  this.directory = path.dirname(dictFilename);
  this.dict = require(dictFilename);
  this.dictionary = new Dictionary(this.dict);
}

CSVDumper.prototype.getNLetterWords = function (n) {
  return this.dictionary.getSolutionForProblem(new Array( n + 1 ).join( CONSTANTS.BLANK ), new RegExp('^.{' + n +'}$'));
}

CSVDumper.prototype.generateCSVFile = function () {
  var lines = [],
    i,
    j,
    k,
    decodedLetter = '',
    decodedWord = '';

  for (i = 2; i <= 15; i += 1) {
    lines = this.getNLetterWords(i).solution;
    for (j = 0; j < lines.length; j += 1) {
      //ines[j] = this.dictionary.decode(lines[j]);
      decodedWord = '';
      for (k = 0; k < lines[j].length; k += 1) {
        decodedLetter = this.dictionary.decode(lines[j][k]);
        if (decodedLetter.length > 1) {
          decodedWord += decodedLetter.toUpperCase();
        } else {
          decodedWord += decodedLetter;
        }
      }
      lines[j] = decodedWord;
    }
    lines.sort(function (a, b) {
      return a.toLocaleLowerCase().localeCompare(b.toLocaleLowerCase());
    });
    console.log(i, lines.length);
    fs.writeFileSync(path.join(this.directory, 'WordList_' + i + '.csv'), lines.join(',\n'));
  }
}

module.exports = CSVDumper;
<commit_msg>Single list, ABC order is still not corrent for Hungarian.<commit_after>'use strict';

var Dictionary = require('./Dictionary');

var CONSTANTS = require('../Constants');
var path = require('path');
var fs = require('fs');

function CSVDumper(dictFilename) {
  this.directory = path.dirname(dictFilename);
  this.dict = require(dictFilename);
  this.dictionary = new Dictionary(this.dict);
}

CSVDumper.prototype.getNLetterWords = function (n) {
  return this.dictionary.getSolutionForProblem(new Array( n + 1 ).join( CONSTANTS.BLANK ), new RegExp('^.{' + n +'}$'));
}

CSVDumper.prototype.generateCSVFile = function () {
  var lines = [],
    allWords = [],
    i,
    j,
    k,
    decodedLetter = '',
    decodedWord = '';

  for (i = 2; i <= 15; i += 1) {
    lines = this.getNLetterWords(i).solution;
    for (j = 0; j < lines.length; j += 1) {
      //ines[j] = this.dictionary.decode(lines[j]);
      allWords.push(this.dictionary.decode(lines[j]).toUpperCase());
      decodedWord = '';
      for (k = 0; k < lines[j].length; k += 1) {
        decodedLetter = this.dictionary.decode(lines[j][k]);
        if (decodedLetter.length > 1) {
          decodedWord += decodedLetter.toUpperCase();
        } else {
          decodedWord += decodedLetter;
        }
      }
      lines[j] = decodedWord;
    }
    lines.sort(function (a, b) {
      return a.localeCompare(b);
    });
    console.log(i, lines.length);
    allWords = allWords.LUnique();
    fs.writeFileSync(path.join(this.directory, 'WordList_' + i + '.csv'), lines.join(',\n'));
  }

  allWords.sort(function (a, b) {
    return a.localeCompare(b);
  });

  fs.writeFileSync(path.join(this.directory, 'WordList.csv'), allWords.join('\n'));
}

module.exports = CSVDumper;

{'fraction_non_alphanumeric': 0.17287817099906044, 'fraction_numerical': 0.006263701847792045, 'mean_word_length': 3.1001283697047497, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3487119', 'n_tokens_mistral': 1128, 'n_tokens_neox': 1054, 'n_words': 215}