<commit_before>var Transform = require('readable-stream').Transform;
var inherits = require('inherits');

inherits(Tokenize, Transform);
module.exports = Tokenize;

var codes = {
    lt: '<'.charCodeAt(0),
    gt: '>'.charCodeAt(0),
    slash: '/'.charCodeAt(0),
    dquote: '"'.charCodeAt(0),
    squote: "'".charCodeAt(0)
};

var strings = {
    endScript: Buffer('</script'),
    endComment: Buffer('-->'),
    comment: Buffer('<!--')
};

function Tokenize () {
    if (!(this instanceof Tokenize)) return new Tokenize;
    Transform.call(this, { objectMode: true });
    this.state = 'text';
    this.quoteState = null;
    this.raw = null;
    this.buffers = [];
    this._last = [];
}

Tokenize.prototype._transform = function (buf, enc, next) {
    var offset = 0;
    for (var i = 0; i < buf.length; i++) {
        var b = buf[i];
        this._last.push(b);
        if (this._last.length > 8) this._last.shift();
        
        if (this.raw) {
            var parts = this._testRaw(buf, offset, i);
            if (parts) {
                this.push([ 'text', parts[0] ]);
                
                if (this.raw === strings.endComment) {
                    this.state = 'text';
                    this.buffers = [];
                    this.push([ 'close', parts[1] ]);
                }
                else {
                    this.state = 'open';
                    this.buffers = [ parts[1] ];
                }
                
                this.raw = null;
                offset = i + 1;
            }
        }
        else if (this.state === 'text' && b === codes.lt) {
            if (i > 0 && i - offset > 0) {
                this.buffers.push(buf.slice(offset, i));
            }
            offset = i;
            this.state = 'open';
            this._pushState('text');
        }
        else if (this.state === 'open' && b === codes.dquote) {
            if (this.quoteState) this.quoteState = null;
            else this.quoteState = 'double';
        }
        else if (this.state === 'open' && b === codes.squote) {
            if (this.quoteState) this.quoteState = null;
            else this.quoteState = 'single';
        }
        else if (this.state === 'open' && b === codes.gt && !this.quoteState) {
            if (i > 0) this.buffers.push(buf.slice(offset, i + 1));
            offset = i + 1;
            this.state = 'text';
            
            if (this._getChar(1) === codes.slash) {
                this._pushState('close');
            }
            else {
                var tag = this._getTag();
                if (tag === 'script') this.raw = strings.endScript;
                this._pushState('open');
            }
        }
        else if (this.state === 'open' && compare(this._last, strings.comment)) {
            this.buffers.push(buf.slice(offset, i + 1));
            offset = i + 1;
            this.state = 'text';
            this.raw = strings.endComment;
            this._pushState('open');
        }
    }
    if (offset < buf.length) this.buffers.push(buf.slice(offset));
    next();
};

Tokenize.prototype._flush = function (next) {
    if (this.state === 'text') this._pushState('text');
    this.push(null);
    next();
};

Tokenize.prototype._pushState = function (ev) {
    if (this.buffers.length === 0) return;
    var buf = Buffer.concat(this.buffers);
    this.buffers = [];
    this.push([ ev, buf ]);
};

Tokenize.prototype._getChar = function (i) {
    var offset = 0;
    for (var j = 0; j < this.buffers.length; j++) {
        var buf = this.buffers[j];
        if (offset + buf.length > i) {
            return buf[i - offset];
        }
        offset += buf;
    }
};

Tokenize.prototype._getTag = function () {
    var offset = 0;
    var tag = '';
    for (var j = 0; j < this.buffers.length; j++) {
        var buf = this.buffers[j];
        for (var k = 0; k < buf.length; k++) {
            if (offset === 0 && k === 0) continue;
            var c = String.fromCharCode(buf[k]);
            if (/[^\w-!]/.test(c)) {
                return tag.toLowerCase();
            }
            else tag += c;
        }
        offset += buf.length;
    }
};

Tokenize.prototype._testRaw = function (buf, offset, index) {
    var raw = this.raw, last = this._last;
    if (!compare(last, raw)) return;
    
    this.buffers.push(buf.slice(offset, index + 1));
    var buf = Buffer.concat(this.buffers);
    var k = buf.length - raw.length;
    return [ buf.slice(0, k), buf.slice(k) ];
};

function compare (a, b) {
    if (a.length < b.length) return false;
    for (var i=a.length-1, j=b.length-1; i > 0 && j > 0; i--, j--) {
        if (a[i] !== b[j]) return false;
    }
    return true;
}
<commit_msg>passing cdata test<commit_after>var Transform = require('readable-stream').Transform;
var inherits = require('inherits');

inherits(Tokenize, Transform);
module.exports = Tokenize;

var codes = {
    lt: '<'.charCodeAt(0),
    gt: '>'.charCodeAt(0),
    slash: '/'.charCodeAt(0),
    dquote: '"'.charCodeAt(0),
    squote: "'".charCodeAt(0)
};

var strings = {
    endScript: Buffer('</script'),
    comment: Buffer('<!--'),
    endComment: Buffer('-->'),
    cdata: Buffer('<![CDATA['),
    endCdata: Buffer(']]>')
};

function Tokenize () {
    if (!(this instanceof Tokenize)) return new Tokenize;
    Transform.call(this, { objectMode: true });
    this.state = 'text';
    this.quoteState = null;
    this.raw = null;
    this.buffers = [];
    this._last = [];
}

Tokenize.prototype._transform = function (buf, enc, next) {
    var offset = 0;
    for (var i = 0; i < buf.length; i++) {
        var b = buf[i];
        this._last.push(b);
        if (this._last.length > 9) this._last.shift();
        
        if (this.raw) {
            var parts = this._testRaw(buf, offset, i);
            if (parts) {
                this.push([ 'text', parts[0] ]);
                
                if (this.raw === strings.endComment
                || this.raw === strings.endCdata) {
                    this.state = 'text';
                    this.buffers = [];
                    this.push([ 'close', parts[1] ]);
                }
                else {
                    this.state = 'open';
                    this.buffers = [ parts[1] ];
                }
                
                this.raw = null;
                offset = i + 1;
            }
        }
        else if (this.state === 'text' && b === codes.lt) {
            if (i > 0 && i - offset > 0) {
                this.buffers.push(buf.slice(offset, i));
            }
            offset = i;
            this.state = 'open';
            this._pushState('text');
        }
        else if (this.state === 'open' && b === codes.dquote) {
            if (this.quoteState) this.quoteState = null;
            else this.quoteState = 'double';
        }
        else if (this.state === 'open' && b === codes.squote) {
            if (this.quoteState) this.quoteState = null;
            else this.quoteState = 'single';
        }
        else if (this.state === 'open' && b === codes.gt && !this.quoteState) {
            if (i > 0) this.buffers.push(buf.slice(offset, i + 1));
            offset = i + 1;
            this.state = 'text';
            
            if (this._getChar(1) === codes.slash) {
                this._pushState('close');
            }
            else {
                var tag = this._getTag();
                if (tag === 'script') this.raw = strings.endScript;
                this._pushState('open');
            }
        }
        else if (this.state === 'open' && compare(this._last, strings.comment)) {
            this.buffers.push(buf.slice(offset, i + 1));
            offset = i + 1;
            this.state = 'text';
            this.raw = strings.endComment;
            this._pushState('open');
        }
        else if (this.state === 'open' && compare(this._last, strings.cdata)) {
            this.buffers.push(buf.slice(offset, i + 1));
            offset = i + 1;
            this.state = 'text';
            this.raw = strings.endCdata;
            this._pushState('open');
        }
    }
    if (offset < buf.length) this.buffers.push(buf.slice(offset));
    next();
};

Tokenize.prototype._flush = function (next) {
    if (this.state === 'text') this._pushState('text');
    this.push(null);
    next();
};

Tokenize.prototype._pushState = function (ev) {
    if (this.buffers.length === 0) return;
    var buf = Buffer.concat(this.buffers);
    this.buffers = [];
    this.push([ ev, buf ]);
};

Tokenize.prototype._getChar = function (i) {
    var offset = 0;
    for (var j = 0; j < this.buffers.length; j++) {
        var buf = this.buffers[j];
        if (offset + buf.length > i) {
            return buf[i - offset];
        }
        offset += buf;
    }
};

Tokenize.prototype._getTag = function () {
    var offset = 0;
    var tag = '';
    for (var j = 0; j < this.buffers.length; j++) {
        var buf = this.buffers[j];
        for (var k = 0; k < buf.length; k++) {
            if (offset === 0 && k === 0) continue;
            var c = String.fromCharCode(buf[k]);
            if (/[^\w-!\[\]]/.test(c)) {
                return tag.toLowerCase();
            }
            else tag += c;
        }
        offset += buf.length;
    }
};

Tokenize.prototype._testRaw = function (buf, offset, index) {
    var raw = this.raw, last = this._last;
    if (!compare(last, raw)) return;
    
    this.buffers.push(buf.slice(offset, index + 1));
    var buf = Buffer.concat(this.buffers);
    var k = buf.length - raw.length;
    return [ buf.slice(0, k), buf.slice(k) ];
};

function compare (a, b) {
    if (a.length < b.length) return false;
    for (var i=a.length-1, j=b.length-1; i > 0 && j > 0; i--, j--) {
        if (lower(a[i]) !== lower(b[j])) return false;
    }
    return true;
}

function lower (n) {
    if (n >= 65 && n <= 90) return n + 32;
    return n;
}

{'fraction_non_alphanumeric': 0.1682469235424652, 'fraction_numerical': 0.007665926971958846, 'mean_word_length': 1.6531977522076533, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1900130', 'n_tokens_mistral': 3203, 'n_tokens_neox': 3102, 'n_words': 732}