<commit_before>import { map } from '../promisify';

// This is how many file reads we can do at once.
const BATCH_SIZE = 5;
const UPLOAD_PATH = '../../upload/';

const fs = require('fs');
const path = require('path');
const Promise = require('bluebird');
const walk = require('walk').walk;
const Queue = require('better-queue');

export default class Files {
  private initialized: boolean;
  private files: {};
  private paths: string[];

  constructor() {
    this.initialized = false;
    this.files = {};
  }

  private ensure(): Promise<any> {
    if (this.initialized) {
      return Promise.resolve();
    }

    return this.init();
  }

  /**
   * Get the file extension.
   */
  private getExt(path: string): string {
    return path.substr(path.indexOf('.') - path.length);
  }

  /**
   * Returns the file path with extension stripped.
   */
  private getGlob(path: string): string {
    return path.substr(0, path.indexOf('.'));
  }

  /**
   * Read a sentence in from a file.
   */
  private process(data: any, cb: Function) {
    fs.readFile(data.path, (err: ErrorEvent, txt: Buffer) => {
      if (err) {
        console.error('Could not read file', data.path, err);
        return cb(err);
      }

      let sentence = txt.toString();
      this.files[data.glob].sentence = sentence;
      cb(null);
    });
  }

  private processBatch(batches: any[], cb: Function) {
    map(this, this.process, batches)
      .then(cb)
      .error((err: ErrorEvent) => {
        console.log('got an error pocessing the batches', err);
      });
  }

  /**
   * Load a list of files from the filesystem.
   */
  init(): Promise<any> {
    // Create our batch processing unit, to keep from overloading
    // on boot.
    let batches = new Queue(this.processBatch.bind(this),
      { batchSize: BATCH_SIZE });

    return new Promise((res: Function, rej: Function) => {
      let walker = walk(path.resolve(__dirname, UPLOAD_PATH));

      walker.on('file', (root, fileStats, next) => {
        let file = path.join(root, fileStats.name);
        let glob = this.getGlob(file);
        let ext = this.getExt(file);

        // Track file gobs and extensions of the voice clips.
        if (!this.files[glob]) {
          this.files[glob] = {
            sentence: null,
            ext: null
          }
        }

        // Text files go into our batch processing queue for later reading.
        if (ext === '.txt') {
          batches.push({
            path: file,
            glob: glob
          });
        } else {
          this.files[glob].ext = ext;
        }

        next();
      });

      walker.on('end', () => {
        this.paths = Object.keys(this.files);
        res();
      });
    });
  }

  /**
   * Grab a random sentence and associated sound file path.
   */
  getRandomClip(): Promise<string[2]> {
    return this.ensure().then(() => {
      let items = this.paths;
      let path = items[Math.floor(Math.random()*items.length)];<commit_msg>forgot to add the null check in the files class<commit_after>import { map } from '../promisify';

// This is how many file reads we can do at once.
const BATCH_SIZE = 5;
const UPLOAD_PATH = '../../upload/';

const fs = require('fs');
const path = require('path');
const Promise = require('bluebird');
const walk = require('walk').walk;
const Queue = require('better-queue');

export default class Files {
  private initialized: boolean;
  private files: {};
  private paths: string[];

  constructor() {
    this.initialized = false;
    this.files = {};
  }

  private ensure(): Promise<any> {
    if (this.initialized) {
      return Promise.resolve();
    }

    return this.init();
  }

  /**
   * Get the file extension.
   */
  private getExt(path: string): string {
    return path.substr(path.indexOf('.') - path.length);
  }

  /**
   * Returns the file path with extension stripped.
   */
  private getGlob(path: string): string {
    return path.substr(0, path.indexOf('.'));
  }

  /**
   * Read a sentence in from a file.
   */
  private process(data: any, cb: Function) {
    fs.readFile(data.path, (err: ErrorEvent, txt: Buffer) => {
      if (err) {
        console.error('Could not read file', data.path, err);
        return cb(err);
      }

      let sentence = txt.toString();
      this.files[data.glob].sentence = sentence;
      cb(null);
    });
  }

  private processBatch(batches: any[], cb: Function) {
    map(this, this.process, batches)
      .then(cb)
      .error((err: ErrorEvent) => {
        console.log('got an error pocessing the batches', err);
      });
  }

  /**
   * Load a list of files from the filesystem.
   */
  init(): Promise<any> {
    // Create our batch processing unit, to keep from overloading
    // on boot.
    let batches = new Queue(this.processBatch.bind(this),
      { batchSize: BATCH_SIZE });

    return new Promise((res: Function, rej: Function) => {
      let walker = walk(path.resolve(__dirname, UPLOAD_PATH));

      walker.on('file', (root, fileStats, next) => {
        let file = path.join(root, fileStats.name);
        let glob = this.getGlob(file);
        let ext = this.getExt(file);

        // Track file gobs and extensions of the voice clips.
        if (!this.files[glob]) {
          this.files[glob] = {
            sentence: null,
            ext: null
          }
        }

        // Text files go into our batch processing queue for later reading.
        if (ext === '.txt') {
          batches.push({
            path: file,
            glob: glob
          });
        } else {
          this.files[glob].ext = ext;
        }

        next();
      });

      walker.on('end', () => {
        this.paths = Object.keys(this.files);
        res();
      });
    });
  }

  /**
   * Grab a random sentence and associated sound file path.
   */
  getRandomClip(): Promise<string[2]> {
    return this.ensure().then(() => {
      if (this.paths.length === 0) {
        return null;
      }

      let items = this.paths;
      let path = items[Math.floor(Math.random()*items.length)];
{'fraction_non_alphanumeric': 0.14926361078934305, 'fraction_numerical': 0.0011583650504716201, 'mean_word_length': 2.206366047745358, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6165568', 'n_tokens_mistral': 1854, 'n_tokens_neox': 1815, 'n_words': 513}