<commit_before>var URLValidator = require('valid-url');
var trumpet = require('trumpet');
var url = require('url');
var base64 = require('js-base64').Base64;
var clientCodes = require('./clientCodeFactory');

function URLConverter(forwardURLPrefix, forwardURL) {
    var _forwardURL = forwardURL;
    var _forwardURLObject = url.parse(_forwardURL);
    
    var tr = trumpet();
    
    tr.select('head', function (elem) {
        var first = true;
        var headStream = elem.createStream();
        headStream.on('data', function (data) {
            if (first) {
                headStream.write('<script>'
                    + clientCodes.urlResolver()
                    + clientCodes.overrideXHR(forwardURLPrefix)
                    + clientCodes.base64URLEncoder()
                    + '</script>\n'
                    + data);
                first = false;
            } else {
                headStream.write(data);
            }
        });
        headStream.on('end', function () {
            headStream.end();
        })
    });
    
    tr.selectAll(
        'a,area,base,link,button,script,embed,iframe,frame,img,input,form,body,blockquote,q,ins,del,object,applet,head',
        function (elem) {
            getURLPropertyNames(elem.name).forEach(function (prop) {
                elem.getAttribute(prop, function (rawURL) {
                    if (!rawURL) return;
                    if (rawURL.substr(0, 5) === 'data:') return;
                    elem.setAttribute(prop, convertToForwardURL(rawURL));
                });
            });
        });

    tr.on('error', function (err) {
        console.dir(err);
    });
    
    return tr;
    
    function convertToForwardURL(rawURL) {
        if (URLValidator.isWebUri(rawURL)) return forwardURLPrefix + base64.encodeURI(rawURL);
        if (rawURL == '#') return rawURL;<commit_msg>fix: rawURL.substr is not a function<commit_after>var URLValidator = require('valid-url');
var trumpet = require('trumpet');
var url = require('url');
var base64 = require('js-base64').Base64;
var clientCodes = require('./clientCodeFactory');

function URLConverter(forwardURLPrefix, forwardURL) {
    var _forwardURL = forwardURL;
    var _forwardURLObject = url.parse(_forwardURL);
    
    var tr = trumpet();
    
    tr.select('head', function (elem) {
        var first = true;
        var headStream = elem.createStream();
        headStream.on('data', function (data) {
            if (first) {
                headStream.write('<script>'
                    + clientCodes.urlResolver()
                    + clientCodes.overrideXHR(forwardURLPrefix)
                    + clientCodes.base64URLEncoder()
                    + '</script>\n'
                    + data);
                first = false;
            } else {
                headStream.write(data);
            }
        });
        headStream.on('end', function () {
            headStream.end();
        })
    });
    
    tr.selectAll(
        'a,area,base,link,button,script,embed,iframe,frame,img,input,form,body,blockquote,q,ins,del,object,applet,head',
        function (elem) {
            getURLPropertyNames(elem.name).forEach(function (prop) {
                elem.getAttribute(prop, function (rawURL) {
                    if (typeof rawURL === 'string') return;
                    if (rawURL.substr(0, 5) === 'data:') return;
                    elem.setAttribute(prop, convertToForwardURL(rawURL));
                });
            });
        });

    tr.on('error', function (err) {
        console.dir(err);
    });
    
    return tr;
    
    function convertToForwardURL(rawURL) {
        if (URLValidator.isWebUri(rawURL)) return forwardURLPrefix + base64.encodeURI(rawURL);
        if (rawURL == '#') return rawURL;
{'fraction_non_alphanumeric': 0.13248542660307366, 'fraction_numerical': 0.006359300476947536, 'mean_word_length': 1.95846394984326, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1831633', 'n_tokens_mistral': 1072, 'n_tokens_neox': 1040, 'n_words': 186}