<commit_before>import {AbstractTokenizer} from "./AbstractTokenizer";
import {EndOfFile} from "./";
import {StreamReader} from "then-read-stream";
import * as Stream from "stream";

export class ReadStreamTokenizer extends AbstractTokenizer {

  private streamReader: StreamReader;

  public constructor(stream: Stream.Readable, fileSize?: number) {
    super();
    this.streamReader = new StreamReader(stream);
    this.fileSize = fileSize;
  }

  /**
   * Read buffer from stream
   * @param buffer
   * @param offset is the offset in the buffer to start writing at; if not provided, start at 0
   * @param length is an integer specifying the number of bytes to read
   * @returns Promise number of bytes read
   */
  public readBuffer(buffer: Buffer | Uint8Array, offset?: number, length?: number): Promise<number> {

    if (!length) {
      length = buffer.length;
    }

    return this.streamReader.read(buffer, offset, length)
      .then((bytesRead) => {
        this.position += bytesRead;
        return bytesRead;
      })
      .catch((err) => {
        if (err === StreamReader.EndOfStream) // Convert EndOfStream into EndOfFile
          throw EndOfFile;
        else throw err;
      });
  }

  /**
   * Peek (read ahead) buffer from tokenizer
   * @param buffer
   * @param offset is the offset in the buffer to start writing at; if not provided, start at 0
   * @param length is an integer specifying the number of bytes to read
   * @param position is an integer specifying where to begin reading from in the file. If position is null, data will be read from the current file position.
   * @returns {Promise<TResult|number>}
   */
  public peekBuffer(buffer: Buffer | Uint8Array, offset?: number, length?: number): Promise<number> {

    if (!length) {
      length = buffer.length;
    }

    return this.streamReader.peek(buffer, offset, length) // ToDo: looks like wrong return type is defined in fs.read
      .catch((err) => {
        if (err === StreamReader.EndOfStream) // Convert EndOfStream into EndOfFile
          throw EndOfFile;
        else throw err;
      });
  }

  public ignore(length: number): Promise<void> {
    const buf = new Buffer(length);
    return this.streamReader.read(buf, 0, length).then(() => null); // Stream cannot skip data
  }
}
<commit_msg>Remove false comment<commit_after>import {AbstractTokenizer} from "./AbstractTokenizer";
import {EndOfFile} from "./";
import {StreamReader} from "then-read-stream";
import * as Stream from "stream";

export class ReadStreamTokenizer extends AbstractTokenizer {

  private streamReader: StreamReader;

  public constructor(stream: Stream.Readable, fileSize?: number) {
    super();
    this.streamReader = new StreamReader(stream);
    this.fileSize = fileSize;
  }

  /**
   * Read buffer from stream
   * @param buffer
   * @param offset is the offset in the buffer to start writing at; if not provided, start at 0
   * @param length is an integer specifying the number of bytes to read
   * @returns Promise number of bytes read
   */
  public readBuffer(buffer: Buffer | Uint8Array, offset?: number, length?: number): Promise<number> {

    if (!length) {
      length = buffer.length;
    }

    return this.streamReader.read(buffer, offset, length)
      .then((bytesRead) => {
        this.position += bytesRead;
        return bytesRead;
      })
      .catch((err) => {
        if (err === StreamReader.EndOfStream) // Convert EndOfStream into EndOfFile
          throw EndOfFile;
        else throw err;
      });
  }

  /**
   * Peek (read ahead) buffer from tokenizer
   * @param buffer
   * @param offset is the offset in the buffer to start writing at; if not provided, start at 0
   * @param length is an integer specifying the number of bytes to read
   * @param position is an integer specifying where to begin reading from in the file. If position is null, data will be read from the current file position.
   * @returns {Promise<TResult|number>}
   */
  public peekBuffer(buffer: Buffer | Uint8Array, offset?: number, length?: number): Promise<number> {

    if (!length) {
      length = buffer.length;
    }

    return this.streamReader.peek(buffer, offset, length)
      .catch((err) => {
        if (err === StreamReader.EndOfStream) // Convert EndOfStream into EndOfFile
          throw EndOfFile;
        else throw err;
      });
  }

  public ignore(length: number): Promise<void> {
    const buf = new Buffer(length);
    return this.streamReader.read(buf, 0, length).then(() => null); // Stream cannot skip data
  }
}

{'fraction_non_alphanumeric': 0.10463838206199165, 'fraction_numerical': 0.0021982853374367993, 'mean_word_length': 3.0954095409540954, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1330787', 'n_tokens_mistral': 1278, 'n_tokens_neox': 1245, 'n_words': 466}