<commit_before>var http = require('http'),
    htmlparser = require('htmlparser');

var find = function(tag, dom) {
    var children = [], c;
    if (Array.isArray(dom)) {
        dom.forEach(function(item) {
            var c = find(tag, item);
            if (c.length) {
                c.forEach(function(i) {
                    children.push(i);
                });
            }
        });
    } else {
        if (dom.type === "tag" && dom.name === tag) {
            children.push(dom);
        }
        
        if (dom.children) {
            c = find(tag, dom.children);
            if (c.length) {
                c.forEach(function(i) {
                    children.push(i);
                });
            }
        }
    }

    return children;
};

var filter = function(items) {
    var json = [];
    items.forEach(function(item) {
        var sub = item.children,
            o = {
                host: '',
                browsers: []
            };

        sub.forEach(function(s) {
            var url, details, title, data;
            if (s.data && s.data.indexOf('listening on') === 0) {
                url = s.data.replace('listening on ', '');
                o.host = url;
            }

            if (s.name && s.name === 'img')  {
                if (!('class' in s.attribs)) {
                    title = s.attribs.title.replace('type=WebDriver', '').replace('{', '').replace('}', '').trim();
                    data = title.trim().split(',');
                    details = {};
                    data.forEach(function(line) {
                        var i = line.trim().split('=');
                        details[i[0]] = i[1];
                    });
                    o.browsers.push(details);
                }
            }
        });

        json.push(o);
    });

    return json;
};

var flatten = function(items) {
    var out = [];
    items.forEach(function(item) {
        item.browsers.forEach(function(o) {
            o.host = item.host;
            out.push(o);
        });<commit_msg>Fixed issue with drivers with no image and disconnected ones<commit_after>var http = require('http'),
    htmlparser = require('htmlparser');

var find = function(tag, dom) {
    var children = [], c;
    if (Array.isArray(dom)) {
        dom.forEach(function(item) {
            var c = find(tag, item);
            if (c.length) {
                c.forEach(function(i) {
                    children.push(i);
                });
            }
        });
    } else {
        if (dom.type === "tag" && dom.name === tag) {
            children.push(dom);
        }
        
        if (dom.children) {
            c = find(tag, dom.children);
            if (c.length) {
                c.forEach(function(i) {
                    children.push(i);
                });
            }
        }
    }

    return children;
};

var filter = function(items) {
    var json = [];
    items.forEach(function(item) {
        var sub = item.children,
            o = {
                host: '',
                browsers: []
            };

        sub.forEach(function(s) {
            var url, details, title, data;
            if (s.data && s.data.indexOf('listening on') === 0) {
                url = s.data.replace('listening on ', '');
                o.host = url;
            }

            if (s.name && (s.name === 'img' || s.name === 'a'))  {
                if (!('class' in s.attribs)) {
                    title = s.attribs.title.replace('type=WebDriver', '').replace('{', '').replace('}', '').trim();
                    data = title.trim().split(',');
                    details = {};
                    data.forEach(function(line) {
                        var i = line.trim().split('=');
                        details[i[0]] = i[1];
                    });
                    delete details.seleniumProtocol;
                    o.browsers.push(details);
                }
            }
        });

        json.push(o);
    });

    return json;
};

var flatten = function(items) {
    var out = [];
    items.forEach(function(item) {
        item.browsers.forEach(function(o) {
            o.host = item.host;
            if (o.host.indexOf('cannot be reached at the moment') === -1) {
                out.push(o);
            }
        });
{'fraction_non_alphanumeric': 0.15845724907063197, 'fraction_numerical': 0.0016263940520446097, 'mean_word_length': 1.1830628803245435, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2913827', 'n_tokens_mistral': 1254, 'n_tokens_neox': 1202, 'n_words': 231}