<commit_before>import Stream from "stream";

export default class AuroraCmdTransformLines extends Stream.Transform {

    constructor() {

        super({objectMode: true});

        this._leftoverData = '';
    }

    _transform(respChunk, encoding, done) {

        respChunk = respChunk.toString();

        if (this._leftoverData) {
            respChunk = this._leftoverData + respChunk;
            this._leftoverData = '';
        }

        let lines = respChunk.split("\n");

        this._leftoverData = lines.pop();

        lines.map(line => {
            return line.trim();
        });

        this.push(_.compact(lines));

        done();
    }

    _flush(done) {
        
        this._leftoverData = this._leftoverData.trim();
        if (this._leftoverData) {
            console.log(this._leftoverData);<commit_msg>Fixed bug where aurora array results wouldn't have carriage returns trimmed.<commit_after>import Stream from "stream";
import _ from 'lodash';

export default class AuroraCmdTransformLines extends Stream.Transform {

    constructor() {

        super({objectMode: true});

        this._leftoverData = '';
    }

    _transform(respChunk, encoding, done) {

        respChunk = respChunk.toString();

        if (this._leftoverData) {
            respChunk = this._leftoverData + respChunk;
            this._leftoverData = '';
        }

        let lines = respChunk.split("\n");

        this._leftoverData = lines.pop();

        lines = lines.map(line => {
            return line.trim();
        });

        this.push(_.compact(lines));

        done();
    }

    _flush(done) {
        
        this._leftoverData = this._leftoverData.trim();
        if (this._leftoverData) {
            console.log(this._leftoverData);
{'fraction_non_alphanumeric': 0.11997736276174306, 'fraction_numerical': 0.0, 'mean_word_length': 2.101754385964912, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4644755', 'n_tokens_mistral': 538, 'n_tokens_neox': 470, 'n_words': 95}