<commit_before>var Stream = require('stream').Stream;

module.exports = function (parser, opts) {
    var stream = new Stream;
    stream.writable = true;
    stream.readable = true;
    
    var selectors = [];
    
    stream.select = function (s, fn) {
        var sel = createSelector(s, fn, opts.special);
        selectors.push(sel);
        return stream;
    };
    
    var updating = false;
    
    stream.pre = function (name, t) {
        if (name === 'open') {
            selectors.forEach(function (sel) {
                sel(t, parser)
                if (sel.removing) updating = true;
            });
        }
        else if (name === 'close') {
            var level = parser.tags.reduce(function (sum, t) {
                return sum + (opts.special.indexOf(t.name) < 0 ? 1 : 0);
            }, 0);
            
            selectors.forEach(function (sel) {
                var up = sel.updating;
                sel.pending = sel.pending.filter(function (p) {
                    var done = level < p.level;
                    if (done) p.callback.call(stream, p.buffered);
                    return !done;
                });
                
                if (up && !sel.updating) {
                    if (sel.removing) {
                        sel.removeOnPost = true;
                    }
                    else updating = false;
                }
            });
        }
    };
    
    stream.post = function (name, t) {
        if (name === 'open') {
            selectors.forEach(function (sel) {
                if (sel.updating) {
                    updating = true;
                }
                sel.pending.forEach(function (p) {
                    p.buffered = '';
                });
            });
            
            for (var i = 0; i < parser.tags.length; i++) {
                var t = parser.tags[i];
                if (opts.special.indexOf(t.name) >= 0) {
                    parser.tags.splice(i, 1);
                    i--;
                }
            }
        }
        else if (name == 'close') {
            selectors.forEach(function (sel) {
                if (sel.removeOnPost) {
                    updating = false;
                    sel.removing = false;
                    sel.removeOnPost = false;
                }
            });
        }
    };
    
    stream.raw = function (s) {
        selectors.forEach(function (sel) {
            sel.pending.forEach(function (p) {
                p.buffered += s;
            });
        });
        
        if (!updating) stream.emit('data', s);
    };
    
    return stream;
};

function createSelector (selector, fn, special) {
    var saveSiblings = false;
    var parts = selector.split(/([\s>+]+)/).map(function (s) {
        if (s.match(/^\s+$/)) return;
        
        var op = s.trim();
        if (op === '+') saveSiblings = true;
        if (op === '>' || op === '+') return { combinator : op };
        
        var m = {
            name : s.match(/^([\w-]+|\*)/),
            class : s.match(/\.([\w-]+)/),
            id : s.match(/#([\w-]+)/),
            pseudo : s.match(/:([\w-]+)/),
            attribute : s.match(/\[([^\]]+)\]/),
        };
        
        return {
            name : m.name && m.name[1].toUpperCase(),
            class : m.class && m.class[1],
            id : m.id && m.id[1],
            pseudo : m.pseudo && m.pseudo[1],
            attribute : m.attribute && m.attribute[1],
        };
    }).filter(Boolean);
    
    var depth = parts.reduce(function (sum, s) {
        return sum + (s.combinator ? 0 : 1);
    }, 0);
    
    var siblings = [];
    
    var sel = function (tag, parser) {
        var tags = parser.tags;
        
        if (!siblings[tags.length]) siblings[tags.length] = [];
        siblings[tags.length].push(tag);
        
        if (depth > tags.length) return;
        
        // hypothesis: the selector matches
        var j = parts.length - 1;
        var i = tags.length - 1;
        
        function check (t, p) {
            // try to falsify the hypothesis on each tag/part match:
            if (p.name !== '*' && p.name && p.name !== t.name) return false;
            if (p.class && p.class !== t.attributes.class) return false;
            if (p.id && p.id !== t.attributes.id) return false;
            return true;
        }
        
        for (; j >= 0; j--, i--) {
            var t = tags[i];
            var p = parts[j];
            if (p.combinator === '>') {
                for (var ii = i; ii >= 0; ii--) {
                    if (check(tags[ii], parts[j-1])) {
                        break;
                    }
                }
                if (ii < 0) return;
                j -= 2;
            }
            else if (p.combinator === '+') {
                var ts = siblings[i + 2];
                t = ts[ts.length - 2];
                if (!t) return;
                p = parts[j - 1];
                if (!check(t, p)) return;
                j -= 2;
            }
            else if (!check(t, p)) return;
        }
        
        var p = { level : tags.length };
        var expired = false;
        function expire () {
            throw new Error('Parse expired. You had your chance.');
        }
        
        fn({
            name : tag.name.toLowerCase(),
            attributes : tag.attributes,
            html : function (cb) {
                if (expired) expire();
                
                p.buffered = '';
                p.callback = cb;
                sel.pending.push(p);
            },
            update : function (cb) {
                if (expired) expire();
                
                p.buffered = '';
                p.callback = function (html) {
                    this.emit('data',
                        typeof cb === 'function' ? cb(html) : cb
                    );
                    sel.updating = false;
                };
                sel.updating = true;
                sel.pending.push(p);
            },
            replace : function (cb) {
                if (expired) expire();
                
                p.buffered = '';
                p.callback = function (html) {
                    this.emit('data',<commit_msg>fix for selecting html with embedded closing tags<commit_after>var Stream = require('stream').Stream;

module.exports = function (parser, opts) {
    var stream = new Stream;
    stream.writable = true;
    stream.readable = true;
    
    var selectors = [];
    
    stream.select = function (s, fn) {
        var sel = createSelector(s, fn, opts.special);
        selectors.push(sel);
        return stream;
    };
    
    var updating = false;
    
    stream.pre = function (name, t) {
        if (name === 'open') {
            selectors.forEach(function (sel) {
                sel(t, parser)
                if (sel.removing) updating = true;
            });
        }
        else if (name === 'close') {
            var level = parser.tags.reduce(function (sum, t) {
                return sum + (opts.special.indexOf(t.name) < 0 ? 1 : 0);
            }, 0);
            
            selectors.forEach(function (sel) {
                var up = sel.updating;
                sel.pending = sel.pending.filter(function (p) {
                    var done = level < p.level;
                    if (done) p.callback.call(stream, p.buffered);
                    return !done;
                });
                
                if (up && !sel.updating) {
                    if (sel.removing) {
                        sel.removeOnPost = true;
                    }
                    else updating = false;
                }
            });
        }
    };
    
    stream.post = function (name, t) {
        if (name === 'open') {
            selectors.forEach(function (sel) {
                if (sel.updating) {
                    updating = true;
                }
                sel.pending.forEach(function (p) {
                    if (p.writes === 0) {
                        p.buffered = '';
                    }
                    if (typeof p.writes === 'number') p.writes ++;
                });
            });
            
            for (var i = 0; i < parser.tags.length; i++) {
                var t = parser.tags[i];
                if (opts.special.indexOf(t.name) >= 0) {
                    parser.tags.splice(i, 1);
                    i--;
                }
            }
        }
        else if (name == 'close') {
            selectors.forEach(function (sel) {
                if (sel.removeOnPost) {
                    updating = false;
                    sel.removing = false;
                    sel.removeOnPost = false;
                }
            });
        }
    };
    
    stream.raw = function (s) {
        selectors.forEach(function (sel) {
            sel.pending.forEach(function (p) {
                p.buffered += s;
            });
        });
        
        if (!updating) stream.emit('data', s);
    };
    
    return stream;
};

function createSelector (selector, fn, special) {
    var saveSiblings = false;
    var parts = selector.split(/([\s>+]+)/).map(function (s) {
        if (s.match(/^\s+$/)) return;
        
        var op = s.trim();
        if (op === '+') saveSiblings = true;
        if (op === '>' || op === '+') return { combinator : op };
        
        var m = {
            name : s.match(/^([\w-]+|\*)/),
            class : s.match(/\.([\w-]+)/),
            id : s.match(/#([\w-]+)/),
            pseudo : s.match(/:([\w-]+)/),
            attribute : s.match(/\[([^\]]+)\]/),
        };
        
        return {
            name : m.name && m.name[1].toUpperCase(),
            class : m.class && m.class[1],
            id : m.id && m.id[1],
            pseudo : m.pseudo && m.pseudo[1],
            attribute : m.attribute && m.attribute[1],
        };
    }).filter(Boolean);
    
    var depth = parts.reduce(function (sum, s) {
        return sum + (s.combinator ? 0 : 1);
    }, 0);
    
    var siblings = [];
    
    var sel = function (tag, parser) {
        var tags = parser.tags;
        
        if (!siblings[tags.length]) siblings[tags.length] = [];
        siblings[tags.length].push(tag);
        
        if (depth > tags.length) return;
        
        // hypothesis: the selector matches
        var j = parts.length - 1;
        var i = tags.length - 1;
        
        function check (t, p) {
            // try to falsify the hypothesis on each tag/part match:
            if (p.name !== '*' && p.name && p.name !== t.name) return false;
            if (p.class && p.class !== t.attributes.class) return false;
            if (p.id && p.id !== t.attributes.id) return false;
            return true;
        }
        
        for (; j >= 0; j--, i--) {
            var t = tags[i];
            var p = parts[j];
            if (p.combinator === '>') {
                for (var ii = i; ii >= 0; ii--) {
                    if (check(tags[ii], parts[j-1])) {
                        break;
                    }
                }
                if (ii < 0) return;
                j -= 2;
            }
            else if (p.combinator === '+') {
                var ts = siblings[i + 2];
                t = ts[ts.length - 2];
                if (!t) return;
                p = parts[j - 1];
                if (!check(t, p)) return;
                j -= 2;
            }
            else if (!check(t, p)) return;
        }
        
        var p = { level : tags.length };
        var expired = false;
        function expire () {
            throw new Error('Parse expired. You had your chance.');
        }
        
        fn({
            name : tag.name.toLowerCase(),
            attributes : tag.attributes,
            html : function (cb) {
                if (expired) expire();
                
                p.buffered = '';
                p.callback = cb;
                p.writes = 0;
                sel.pending.push(p);
            },
            update : function (cb) {
                if (expired) expire();
                
                p.buffered = '';
                p.callback = function (html) {
                    this.emit('data',
                        typeof cb === 'function' ? cb(html) : cb
                    );
                    sel.updating = false;
                };
                p.writes = 0;
                sel.updating = true;
                sel.pending.push(p);
            },
            replace : function (cb) {
                if (expired) expire();
                
                p.buffered = '';
                p.callback = function (html) {
                    this.emit('data',
{'fraction_non_alphanumeric': 0.14282343688556823, 'fraction_numerical': 0.004325599685410932, 'mean_word_length': 1.0293648260453239, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7434912', 'n_tokens_mistral': 3672, 'n_tokens_neox': 3467, 'n_words': 828}