<commit_before>
var util = require('util'),
Transform = require('stream').Transform;

util.inherits(Aline, Transform);

function Aline(options) {
    if (!(this instanceof Aline))
     return new Aline(options);

    Transform.call(this, options);
    this._tail = Buffer.alloc(0);
    this._separator = (options && options.separator) || '\n';
}

Aline.prototype._transform = function (chunk, encoding, callback) {

    var index = chunk.lastIndexOf(this._separator);

    if (index === chunk.length) {
        this._tail = Buffer.alloc(0);
        return callback(null, chunk);
    }

    var head = Buffer.concat([this._tail, chunk.slice(0, index)]);
    this._tail = Buffer.from(chunk.slice(index + 1));

    callback(null, head);
};

Aline.prototype._flush = function(callback) {
    callback(null, this._tail);
};

module.exports = Aline;<commit_msg>update to es6 format<commit_after>
const Transform = require('stream').Transform;

class Aline extends Transform {
    constructor(options) {
        super();

        this._tail = Buffer.alloc(0);
        this._separator = (options && options.separator) || '\n';
    }
    
    _transform(chunk, encoding, callback) {
        const index = chunk.lastIndexOf(this._separator);
        
        if (index === chunk.length || index === -1) {
            callback(null, Buffer.concat([this._tail, chunk]));
            this._tail = Buffer.alloc(0);
            return;
        }

        const head = Buffer.concat([this._tail, chunk.slice(0, index)]);
        this._tail = Buffer.from(chunk.slice(index + 1));

        callback(null, head);
    }
    
    _flush(callback) {
        callback(null, this._tail);
    }
}


module.exports = Aline;
{'fraction_non_alphanumeric': 0.1502651738361815, 'fraction_numerical': 0.0058927519151443725, 'mean_word_length': 2.8503401360544216, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2379731', 'n_tokens_mistral': 520, 'n_tokens_neox': 497, 'n_words': 107}