<commit_before>var Stream = require('stream');
var EventEmitter = require('events').EventEmitter;
var split = require('split');

var re = {
    ok: new RegExp([
        '^(not )?ok\\b(?:',
        '\\s+(\\d+)(?:\\s+(?:(?:\\s*-\\s*)?(.*)))?',
        ')?'
    ].join('')),
    plan: /^(\d+)\.\.(\d+)\b/,
    comment: /^#\s*(.+)/,
    version: /^TAP\s+version\s+(\d+)/i
};

function writable (s) {
    var stream = new Stream;
    stream.writable = true;
    stream.write = function (buf) { return s.write(buf) };
    stream.end = function (buf) { return s.end(buf) };
    return stream;
}

module.exports = function (cb) {
    var sp = split();
    var stream = writable(sp);
    
    var results = stream.results = {
        ok: undefined,
        asserts: [],
        pass: [],
        fail: [],
        errors: []
    };
    
    stream.on('assert', function (res) {
        results.asserts.push(res);
        if (!res.ok) results.ok = false;
        (res.ok ? results.pass : results.fail).push(res);
        
        var prev = results.asserts[results.asserts.length - 2];
        if (prev && prev.number + 1 !== res.number) {
            stream.emit('parseError', {
                message: 'assert out of order'
            });
        }
    });
    
    stream.on('plan', function (plan) {
        if (results.plan !== undefined) {
            stream.emit('parseError', {
                message: 'unexpected additional plan',
            });
            return;
        }
        results.plan = plan;
        checkAssertionStart();
    });
    
    var planMismatch = false;
    function checkAssertionStart () {
        if (planMismatch) return;
        if (!results.asserts[0]) return;
        if (!results.plan) return;
        if (results.asserts[0].number === results.plan.start) return;
        
        planMismatch = true;
        stream.emit('parseError', {
            message: 'plan range mismatch'
        });
    }
    
    stream.on('parseError', function (err) {
        err.line = lineNum;
        results.errors.push(err);
    });
    
    var lineNum = 0;
    var ended = false;
    
    sp.on('data', ondata);
    sp.on('end', onend);
    sp.on('close', onend);
    
    if (cb) stream.on('results', cb);
    
    return stream;
    
    function onend () {
        if (ended) return;
        ended = true;
        
        if (results.asserts.length === 0) {
            stream.emit('parseError', {
                message: 'no assertions found'
            });
        }
        
        if (results.plan === undefined) {
            stream.emit('parseError', {
                message: 'no plan found'
            });
        }
        if (results.errors.length > 0) results.ok = false;
        if (results.ok === undefined) results.ok = true;
        
        stream.emit('results', results);
    }
    
    function ondata (line) {
        var m;
        lineNum ++;
        
        if (m = re.version.exec(line)) {
            var ver = /^\d+(\.\d*)?$/.test(m[1]) ? Number(m[1]) : m[1];
            stream.emit('version', ver);
        }
        else if (m = re.comment.exec(line)) {
            stream.emit('comment', m[1]);
        }
        else if (m = re.ok.exec(line)) {
            var ok = !m[1];
            var num = m[2] && Number(m[2]);
            var name = m[3];<commit_msg>passing not enough tests test<commit_after>var Stream = require('stream');
var EventEmitter = require('events').EventEmitter;
var split = require('split');

var re = {
    ok: new RegExp([
        '^(not )?ok\\b(?:',
        '\\s+(\\d+)(?:\\s+(?:(?:\\s*-\\s*)?(.*)))?',
        ')?'
    ].join('')),
    plan: /^(\d+)\.\.(\d+)\b/,
    comment: /^#\s*(.+)/,
    version: /^TAP\s+version\s+(\d+)/i
};

function writable (s) {
    var stream = new Stream;
    stream.writable = true;
    stream.write = function (buf) { return s.write(buf) };
    stream.end = function (buf) { return s.end(buf) };
    return stream;
}

module.exports = function (cb) {
    var sp = split();
    var stream = writable(sp);
    
    var results = stream.results = {
        ok: undefined,
        asserts: [],
        pass: [],
        fail: [],
        errors: []
    };
    
    stream.on('assert', function (res) {
        results.asserts.push(res);
        if (!res.ok) results.ok = false;
        (res.ok ? results.pass : results.fail).push(res);
        
        var prev = results.asserts[results.asserts.length - 2];
        if (prev && prev.number + 1 !== res.number) {
            stream.emit('parseError', {
                message: 'assert out of order'
            });
        }
    });
    
    stream.on('plan', function (plan) {
        if (results.plan !== undefined) {
            stream.emit('parseError', {
                message: 'unexpected additional plan',
            });
            return;
        }
        results.plan = plan;
        checkAssertionStart();
    });
    
    var planMismatch = false;
    function checkAssertionStart () {
        if (planMismatch) return;
        if (!results.asserts[0]) return;
        if (!results.plan) return;
        if (results.asserts[0].number === results.plan.start) return;
        
        planMismatch = true;
        stream.emit('parseError', {
            message: 'plan range mismatch'
        });
    }
    
    stream.on('parseError', function (err) {
        results.ok = false;
        err.line = lineNum;
        results.errors.push(err);
    });
    
    var lineNum = 0;
    var ended = false;
    
    sp.on('data', ondata);
    sp.on('end', onend);
    sp.on('close', onend);
    
    if (cb) stream.on('results', cb);
    
    return stream;
    
    function onend () {
        if (ended) return;
        ended = true;
        
        if (results.asserts.length === 0) {
            stream.emit('parseError', {
                message: 'no assertions found'
            });
        }
        
        if (results.plan === undefined) {
            stream.emit('parseError', {
                message: 'no plan found'
            });
        }
        if (results.ok === undefined) results.ok = true;
        
        var last = results.asserts[results.asserts.length - 1].number;
        if (results.ok && last < results.plan.end) {
            stream.emit('parseError', {
                message: 'not enough tests'
            });
        }
        
        stream.emit('results', results);
    }
    
    function ondata (line) {
        var m;
        lineNum ++;
        
        if (m = re.version.exec(line)) {
            var ver = /^\d+(\.\d*)?$/.test(m[1]) ? Number(m[1]) : m[1];
            stream.emit('version', ver);
        }
        else if (m = re.comment.exec(line)) {
            stream.emit('comment', m[1]);
        }
        else if (m = re.ok.exec(line)) {
            var ok = !m[1];
            var num = m[2] && Number(m[2]);
            var name = m[3];
{'fraction_non_alphanumeric': 0.17142025611175785, 'fraction_numerical': 0.004365541327124563, 'mean_word_length': 1.6053828658074298, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1098793', 'n_tokens_mistral': 2164, 'n_tokens_neox': 2146, 'n_words': 461}