<commit_before>(function () {
  "use strict";

  var fs = require('fs'),
    StringDecoder = require('string_decoder').StringDecoder;

  function LineReader(fd, cb, separator, encoding, bufferSize) {
    var filePosition = 0,
      encoding = encoding || 'utf8',
      separator = separator || '\n',
      bufferSize = bufferSize || 1024,
      buffer = new Buffer(bufferSize),
      bufferStr = '',
      decoder = new StringDecoder(encoding),
      closed = false,
      eof = false,
      separatorIndex = -1;

    function close() {
      if (!closed) {
        fs.close(fd, function (err) {
          if (err) {
            throw err;
          }
        });
        closed = true;
      }
    }

    function readToSeparator(cb) {
      function readChunk() {
        fs.read(fd, buffer, 0, bufferSize, filePosition, function (err, bytesRead) {
          var separatorAtEnd;

          if (err) {
            throw err;
          }

          if (bytesRead < bufferSize) {
            eof = true;
            close();
          }

          filePosition += bytesRead;

          bufferStr += decoder.write(buffer.slice(0, bytesRead));

          if (separatorIndex < 0) {
            separatorIndex = bufferStr.indexOf(separator);
          }

          separatorAtEnd = separatorIndex === bufferStr.length - 1;
          if (bytesRead && (separatorIndex === -1 || separatorAtEnd) && !eof) {
            readChunk();
          } else {
            cb();
          }
        });
      }

      readChunk();
    }

    function hasNextLine() {
      return bufferStr.length > 0 || !eof;
    }

    function nextLine(cb) {
      function getLine() {
        var ret = bufferStr.substring(0, separatorIndex - 1);

        bufferStr = bufferStr.substring(separatorIndex + separator.length);
        separatorIndex = -1;
        cb(ret);
      }

      if (separatorIndex < 0) {
        separatorIndex = bufferStr.indexOf(separator);
      }

      if (separatorIndex < 0) {
        if (eof) {
          if (hasNextLine()) {
            separatorIndex = bufferStr.length;
            getLine();
          } else {
            throw new Error('No more lines to read.');
          }
        } else {
          readToSeparator(getLine);
        }
      } else {
        getLine();
      }<commit_msg>reset to initial state to test<commit_after>(function () {
  "use strict";

  var fs = require('fs'),
    StringDecoder = require('string_decoder').StringDecoder;

  function LineReader(fd, cb, separator, encoding, bufferSize) {
    var filePosition = 0,
      encoding = encoding || 'utf8',
      separator = separator || '\n',
      bufferSize = bufferSize || 1024,
      buffer = new Buffer(bufferSize),
      bufferStr = '',
      decoder = new StringDecoder(encoding),
      closed = false,
      eof = false,
      separatorIndex = -1;

    function close() {
      if (!closed) {
        fs.close(fd, function (err) {
          if (err) {
            throw err;
          }
        });
        closed = true;
      }
    }

    function readToSeparator(cb) {
      function readChunk() {
        fs.read(fd, buffer, 0, bufferSize, filePosition, function (err, bytesRead) {
          var separatorAtEnd;

          if (err) {
            throw err;
          }

          if (bytesRead < bufferSize) {
            eof = true;
            close();
          }

          filePosition += bytesRead;

          bufferStr += decoder.write(buffer.slice(0, bytesRead));

          if (separatorIndex < 0) {
            separatorIndex = bufferStr.indexOf(separator);
          }

          separatorAtEnd = separatorIndex === bufferStr.length - 1;
          if (bytesRead && (separatorIndex === -1 || separatorAtEnd) && !eof) {
            readChunk();
          } else {
            cb();
          }
        });
      }

      readChunk();
    }

    function hasNextLine() {
      return bufferStr.length > 0 || !eof;
    }

    function nextLine(cb) {
      function getLine() {
        var ret = bufferStr.substring(0, separatorIndex);

        bufferStr = bufferStr.substring(separatorIndex + separator.length);
        separatorIndex = -1;
        cb(ret);
      }

      if (separatorIndex < 0) {
        separatorIndex = bufferStr.indexOf(separator);
      }

      if (separatorIndex < 0) {
        if (eof) {
          if (hasNextLine()) {
            separatorIndex = bufferStr.length;
            getLine();
          } else {
            throw new Error('No more lines to read.');
          }
        } else {
          readToSeparator(getLine);
        }
      } else {
        getLine();
      }
{'fraction_non_alphanumeric': 0.11916648578250488, 'fraction_numerical': 0.00759713479487736, 'mean_word_length': 1.655907780979827, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5959653', 'n_tokens_mistral': 1345, 'n_tokens_neox': 1246, 'n_words': 300}