<commit_before>var fs     = require("fs");
var http   = require("http");
var os     = require("os");
var Stream = require("stream");
var URL    = require("url");

// The starting point is everything that works in any environment (browser OR
// Node.js)
var Lazy = require("./lazy.js");

/**
 * @constructor
 */
function StreamedSequence(stream) {
  this.stream = stream;
}

StreamedSequence.prototype = new Lazy.StreamLikeSequence();

StreamedSequence.prototype.openStream = function(callback) {
  this.stream.resume();
  callback(this.stream);
};

/**
 * Handles every chunk of data in this sequence.
 *
 * @param {function(string):*} fn The function to call on each chunk of data as
 *     it's read from the stream. Return false from the function to stop reading
 *     the stream.
 */
StreamedSequence.prototype.each = function(fn) {
  var encoding = this.encoding || "utf-8";

  this.openStream(function(stream) {
    var listener = function(e) {
      if (fn(e) === false) {
        stream.removeListener("data", listener);
      }
    };

    if (stream.setEncoding) {
      stream.setEncoding(encoding);
    }

    stream.on("data", listener);
  });
};

function StreamedLineSequence(parent) {
  this.parent = parent;
}

StreamedLineSequence.prototype = new Lazy.Sequence();

/**
 * Handles every line of data in the underlying file.
 *
 * @param {function(string):*} fn The function to call on each line of data as
 *     it's read from the file. Return false from the function to stop reading
 *     the file.
 */
StreamedLineSequence.prototype.each = function(fn) {
  var i = 0;

  this.parent.each(function(data) {
    var finished = false;

    // TODO: I'm pretty sure there's a bug here: if/when the buffer ends in the
    // middle of a line, this will artificially split that line in two. I'll
    // come back to this later.
    Lazy(data).split(os.EOL || "\n").each(function(line) {
      if (fn(line, i++) === false) {
        finished = true;
        return false;
      }
    });

    if (finished) {
      return false;
    }
  });
};

/**
 * Creates a {@link Sequence} of lines as they are read from a file.
 *
 * @return {Sequence} A sequence comprising the lines in the underlying file, as
 *     they are read.
 */
StreamedSequence.prototype.lines = function() {
  return new StreamedLineSequence(this);
};

function FileStreamSequence(path, encoding) {
  this.path = path;
  this.encoding = encoding;<commit_msg>moved StreamedLineSequence down<commit_after>var fs     = require("fs");
var http   = require("http");
var os     = require("os");
var Stream = require("stream");
var URL    = require("url");

// The starting point is everything that works in any environment (browser OR
// Node.js)
var Lazy = require("./lazy.js");

/**
 * @constructor
 */
function StreamedSequence(stream) {
  this.stream = stream;
}

StreamedSequence.prototype = new Lazy.StreamLikeSequence();

StreamedSequence.prototype.openStream = function(callback) {
  this.stream.resume();
  callback(this.stream);
};

/**
 * Handles every chunk of data in this sequence.
 *
 * @param {function(string):*} fn The function to call on each chunk of data as
 *     it's read from the stream. Return false from the function to stop reading
 *     the stream.
 */
StreamedSequence.prototype.each = function(fn) {
  var encoding = this.encoding || "utf-8";

  this.openStream(function(stream) {
    var listener = function(e) {
      if (fn(e) === false) {
        stream.removeListener("data", listener);
      }
    };

    if (stream.setEncoding) {
      stream.setEncoding(encoding);
    }

    stream.on("data", listener);
  });
};

/**
 * Creates a {@link Sequence} of lines as they are read from a file.
 *
 * @return {Sequence} A sequence comprising the lines in the underlying file, as
 *     they are read.
 */
StreamedSequence.prototype.lines = function() {
  return new StreamedLineSequence(this);
};

/**
 * @constructor
 */
function StreamedLineSequence(parent) {
  this.parent = parent;
}

StreamedLineSequence.prototype = new Lazy.Sequence();

/**
 * Handles every line of data in the underlying file.
 *
 * @param {function(string):*} fn The function to call on each line of data as
 *     it's read from the file. Return false from the function to stop reading
 *     the file.
 */
StreamedLineSequence.prototype.each = function(fn) {
  var i = 0;

  this.parent.each(function(data) {
    var finished = false;

    // TODO: I'm pretty sure there's a bug here: if/when the buffer ends in the
    // middle of a line, this will artificially split that line in two. I'll
    // come back to this later.
    Lazy(data).split(os.EOL || "\n").each(function(line) {
      if (fn(line, i++) === false) {
        finished = true;
        return false;
      }
    });

    if (finished) {
      return false;
    }
  });
};

function FileStreamSequence(path, encoding) {
  this.path = path;
  this.encoding = encoding;
{'fraction_non_alphanumeric': 0.13023161316537993, 'fraction_numerical': 0.0008126777732629012, 'mean_word_length': 3.133501259445844, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2299037', 'n_tokens_mistral': 1479, 'n_tokens_neox': 1447, 'n_words': 478}