<commit_before>'use strict';

let Corpora = function() {

  if(!(this instanceof Corpora)) {
    return new Corpora();
  }
  // https://gist.github.com/VinGarcia/ba278b9460500dad1f50
  // List all files in a directory in Node.js recursively in a synchronous fashion
  let walkSync = function(dir, filelist) {

    if (dir[dir.length-1] != `/`) { dir = dir.concat(`/`); }

    var files = fs.readdirSync(dir);
    filelist = filelist || [];
    if (dir.indexOf(`###`) === -1) {
      files.forEach(function(file) {
        if (fs.statSync(dir + file).isDirectory()) {
          filelist = walkSync(dir + file + `/`, filelist);
        }
        else {
          filelist.push(dir + file);
        }
      });
    }
    return filelist;
  };

  let fs = require(`fs`),
      path = require(`path`),
      zipkit = require(`node-zipkit`),
      mkdirp = require(`mkdirp`),
      unzipLoc = path.join(__dirname, `./unzip-temp`),
      debreak = require(`./lib/debreak`),
      textutil = require(`./lib/textutil`),
      root = path.join(__dirname, `./corpus`),
      books = walkSync(root),
      texts = [],
      // uncompress is poorly named and/or poorly written
      // it returns a filename, uncompressing it as a side-effect
      uncompress = function(filename) {
        if (filename.match(/.(zip|7z)$/)) {
          let unzipName = filename.replace(root, unzipLoc).replace(/\.(zip|7z)$/, ``),
              targPath = path.dirname(unzipName);
          if (!fs.existsSync(unzipName)) {
            if (!fs.existsSync(targPath)) {
              mkdirp(targPath);
            }
            zipkit.unzipSync(`"${filename}"`, targPath);
          }
          filename = unzipName;
        }
        return filename;
      },
      cleanName = (name) => name.replace(/.(txt|js|zip)$/g, ``)
        .replace(root, ``)
        .replace(/^\/|\\/, ``),
      gettext = function(filename) {
        // if zip file, unzip it, and retrieve the zipped file
        // if unzip folder does not exist, create it
        // if unzipped file already exists, do not unzip!
        filename = uncompress(filename);
        let text = fs.readFileSync(filename),
            iconv = require(`iconv-lite`),
            book = iconv.decode(new Buffer.from(text), `ISO=8859-1`);
        // discard windows encoding thingy
        if (book.charCodeAt(0) === 0xFEFF) {
          book = book.slice(1);
        }
        return debreak(book);
      };
<commit_msg>Buffer tweak<commit_after>'use strict';

let Corpora = function() {

  if(!(this instanceof Corpora)) {
    return new Corpora();
  }
  // https://gist.github.com/VinGarcia/ba278b9460500dad1f50
  // List all files in a directory in Node.js recursively in a synchronous fashion
  let walkSync = function(dir, filelist) {

    if (dir[dir.length-1] != `/`) { dir = dir.concat(`/`); }

    var files = fs.readdirSync(dir);
    filelist = filelist || [];
    if (dir.indexOf(`###`) === -1) {
      files.forEach(function(file) {
        if (fs.statSync(dir + file).isDirectory()) {
          filelist = walkSync(dir + file + `/`, filelist);
        }
        else {
          filelist.push(dir + file);
        }
      });
    }
    return filelist;
  };

  let fs = require(`fs`),
      path = require(`path`),
      zipkit = require(`node-zipkit`),
      mkdirp = require(`mkdirp`),
      unzipLoc = path.join(__dirname, `./unzip-temp`),
      debreak = require(`./lib/debreak`),
      textutil = require(`./lib/textutil`),
      root = path.join(__dirname, `./corpus`),
      books = walkSync(root),
      texts = [],
      // uncompress is poorly named and/or poorly written
      // it returns a filename, uncompressing it as a side-effect
      uncompress = function(filename) {
        if (filename.match(/.(zip|7z)$/)) {
          let unzipName = filename.replace(root, unzipLoc).replace(/\.(zip|7z)$/, ``),
              targPath = path.dirname(unzipName);
          if (!fs.existsSync(unzipName)) {
            if (!fs.existsSync(targPath)) {
              mkdirp(targPath);
            }
            zipkit.unzipSync(`"${filename}"`, targPath);
          }
          filename = unzipName;
        }
        return filename;
      },
      cleanName = (name) => name.replace(/.(txt|js|zip)$/g, ``)
        .replace(root, ``)
        .replace(/^\/|\\/, ``),
      gettext = function(filename) {
        // if zip file, unzip it, and retrieve the zipped file
        // if unzip folder does not exist, create it
        // if unzipped file already exists, do not unzip!
        filename = uncompress(filename);
        let text = fs.readFileSync(filename),
            iconv = require(`iconv-lite`),
            book = iconv.decode(Buffer.from(text), `ISO8859-1`);
        // discard windows encoding thingy
        if (book.charCodeAt(0) === 0xFEFF) {
          book = book.slice(1);
        }
        return debreak(book);
      };

{'fraction_non_alphanumeric': 0.15874641540352313, 'fraction_numerical': 0.010241704219582138, 'mean_word_length': 2.0924635845471817, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '386451', 'n_tokens_mistral': 1567, 'n_tokens_neox': 1525, 'n_words': 336}