<commit_before>var fs = require('fs')
  , request = require('request')
  , url = require('url')
  , https = require('https')
  , http = require('http')
  , EventEmitter = require('events').EventEmitter
  , _allowNetConnect = true
  , _allowLocalConnect = true
  , interceptedUris = {}
  , ignoredUris = {}
  ;

function interceptable(uri, method) {

    if(typeof method === "undefined")
    {
        method = "GET";
    }

    uri = parseUrl(uri);
    if (interceptedUris[uri]) {
        return true;
    }
    if (ignoredUris[uri]) {
        return false;
    }
    if (allowNetConnect === false) {
        if (uri) {
            var hostname = url.parse(uri).hostname
              , requestIsLocal = (hostname == "localhost" || hostname == "127.0.0.1")
              ;
            if (allowLocalConnect === true && requestIsLocal) {
                return false;
            }
            console.error("FAKEWEB: Unhandled" + method + "request to " + uri);
            throw "FAKEWEB: Unhandled " + method + " request to " + uri;
        } else {
            console.error("FAKEWEB: Invalid request");
            throw "FAKEWEB: Invalid request";
        }
    } else {
        return false;
    }
}

function getStatusCode(uri) {
    var statusCode = interceptedUris[uri].statusCode;

    if (Array.isArray(statusCode)) {
        if (statusCode.length === 0) {
            statusCode = 200; // This should not happen but better safe than sorry
        } else if (statusCode.length === 1) {
             statusCode = statusCode[0];
        } else {
            statusCode = statusCode.shift();
        }
    }

    return statusCode;
}

function httpModuleRequest(uri, callback) {
    uri = parseUrl(uri);
    var thisRequest = new EventEmitter();
    thisRequest.setEncoding = function() {};

    var ended = false;
    thisRequest.end = function() {
        if (ended) {
            return;
        }

        ended = true;
        var thisResponse = new EventEmitter();
        // Request module checks against the connection object event emitter
        thisResponse.connection = thisResponse;
        thisResponse.pause = thisResponse.resume = function(){};
        thisResponse.setEncoding = function() {};
        thisResponse.pipe = function(outputStream) {
            outputStream.write(interceptedUris[uri].response);
            outputStream.end();
            return outputStream; // support chaining
        };
        thisResponse.statusCode = getStatusCode(uri);
        thisResponse.headers = interceptedUris[uri].headers;
        if (interceptedUris[uri].contentType) {
            thisResponse.headers['content-type'] = interceptedUris[uri].contentType;
        }
        thisRequest.emit('response', thisResponse);

        //Compatibility with scraperjs
        thisRequest.request = {
            href: uri
        };

        if (callback) {
            callback(thisResponse);
        }

        thisResponse.emit('data', interceptedUris[uri].response);
        thisResponse.emit('end');
        thisResponse.emit('close');

    }
    thisRequest.write = function() {}
    thisRequest.setTimeout = function() {};

    setTimeout(function() {
        if (!ended) {
            thisRequest.end();
        }
    }, 10);

    return thisRequest;
}

function Fakeweb() {
    this.allowNetConnect = _allowNetConnect;
    this.allowLocalConnect = _allowLocalConnect;

    var oldRequestGet = request.get;
    request.get = function(options, callback) {
        if(typeof options === "string"){
            options = {uri: options};
        }

        var uri = options.uri || options.url;
        uri = parseUrl(uri);
        var followRedirect = options.followRedirect !== undefined ? options.followRedirect : true
        if (interceptable(uri)) {
            var statusCode = getStatusCode(uri);

            if (statusCode >= 300 && statusCode < 400 && interceptedUris[uri].headers.Location && followRedirect) {
                var redirectTo = url.resolve(uri, interceptedUris[uri].headers.Location);
                return request.get({uri: redirectTo}, callback);
            } else {
                var resp = {statusCode : statusCode};
                resp.headers = interceptedUris[uri].headers;
                if (interceptedUris[uri].contentType) {
                    resp.headers['content-type'] =  interceptedUris[uri].contentType;
                }
                resp.request = {
                    href: url
                };
                return callback(null, resp, interceptedUris[uri].response);
            }
        } else {
            return oldRequestGet.call(request, options, callback);
        }
    }

    var oldRequestPost = request.post;
    request.post = function(options, callback) {
        if(typeof options === "string"){
            options = {uri: options};
        }

        var url = options.uri || options.url;
        url = parseUrl(url);
        if (interceptable(url, "POST")) {
            var resp = {statusCode : getStatusCode(url)};
            resp.headers = interceptedUris[url].headers;
            if (interceptedUris[url].contentType) {
                resp.headers['content-type'] =  interceptedUris[url].contentType;
            }
            resp.request = {
                href: url
            };
            return callback(null, resp, interceptedUris[url].response);<commit_msg>Make callbacks asynchronous with 2ms setTimeout<commit_after>var fs = require('fs')
  , request = require('request')
  , url = require('url')
  , https = require('https')
  , http = require('http')
  , EventEmitter = require('events').EventEmitter
  , _allowNetConnect = true
  , _allowLocalConnect = true
  , interceptedUris = {}
  , ignoredUris = {}
  ;

function interceptable(uri, method) {

    if(typeof method === "undefined")
    {
        method = "GET";
    }

    uri = parseUrl(uri);
    if (interceptedUris[uri]) {
        return true;
    }
    if (ignoredUris[uri]) {
        return false;
    }
    if (allowNetConnect === false) {
        if (uri) {
            var hostname = url.parse(uri).hostname
              , requestIsLocal = (hostname == "localhost" || hostname == "127.0.0.1")
              ;
            if (allowLocalConnect === true && requestIsLocal) {
                return false;
            }
            console.error("FAKEWEB: Unhandled" + method + "request to " + uri);
            throw "FAKEWEB: Unhandled " + method + " request to " + uri;
        } else {
            console.error("FAKEWEB: Invalid request");
            throw "FAKEWEB: Invalid request";
        }
    } else {
        return false;
    }
}

function getStatusCode(uri) {
    var statusCode = interceptedUris[uri].statusCode;

    if (Array.isArray(statusCode)) {
        if (statusCode.length === 0) {
            statusCode = 200; // This should not happen but better safe than sorry
        } else if (statusCode.length === 1) {
             statusCode = statusCode[0];
        } else {
            statusCode = statusCode.shift();
        }
    }

    return statusCode;
}

function httpModuleRequest(uri, callback) {
    uri = parseUrl(uri);
    var thisRequest = new EventEmitter();
    thisRequest.setEncoding = function() {};

    var ended = false;
    thisRequest.end = function() {
        if (ended) {
            return;
        }

        ended = true;
        var thisResponse = new EventEmitter();
        // Request module checks against the connection object event emitter
        thisResponse.connection = thisResponse;
        thisResponse.pause = thisResponse.resume = function(){};
        thisResponse.setEncoding = function() {};
        thisResponse.pipe = function(outputStream) {
            outputStream.write(interceptedUris[uri].response);
            outputStream.end();
            return outputStream; // support chaining
        };
        thisResponse.statusCode = getStatusCode(uri);
        thisResponse.headers = interceptedUris[uri].headers;
        if (interceptedUris[uri].contentType) {
            thisResponse.headers['content-type'] = interceptedUris[uri].contentType;
        }
        thisRequest.emit('response', thisResponse);

        //Compatibility with scraperjs
        thisRequest.request = {
            href: uri
        };

        setTimeout(function() {
            if (callback) {
                callback(thisResponse);
            }

            thisResponse.emit('data', interceptedUris[uri].response);
            thisResponse.emit('end');
            thisResponse.emit('close');
        }, 2);

    }
    thisRequest.write = function() {}
    thisRequest.setTimeout = function() {};

    setTimeout(function() {
        if (!ended) {
            thisRequest.end();
        }
    }, 10);

    return thisRequest;
}

function Fakeweb() {
    this.allowNetConnect = _allowNetConnect;
    this.allowLocalConnect = _allowLocalConnect;

    var oldRequestGet = request.get;
    request.get = function(options, callback) {
        if(typeof options === "string"){
            options = {uri: options};
        }

        var uri = options.uri || options.url;
        uri = parseUrl(uri);
        var followRedirect = options.followRedirect !== undefined ? options.followRedirect : true
        if (interceptable(uri)) {
            var statusCode = getStatusCode(uri);

            if (statusCode >= 300 && statusCode < 400 && interceptedUris[uri].headers.Location && followRedirect) {
                var redirectTo = url.resolve(uri, interceptedUris[uri].headers.Location);
                return request.get({uri: redirectTo}, callback);
            } else {
                var resp = {statusCode : statusCode};
                resp.headers = interceptedUris[uri].headers;
                if (interceptedUris[uri].contentType) {
                    resp.headers['content-type'] =  interceptedUris[uri].contentType;
                }
                resp.request = {
                    href: url
                };
                setTimeout(function() {
                    callback(null, resp, interceptedUris[uri].response);
                }, 2);
                return;
            }
        } else {
            return oldRequestGet.call(request, options, callback);
        }
    }

    var oldRequestPost = request.post;
    request.post = function(options, callback) {
        if(typeof options === "string"){
            options = {uri: options};
        }

        var url = options.uri || options.url;
        url = parseUrl(url);
        if (interceptable(url, "POST")) {
            var resp = {statusCode : getStatusCode(url)};
            resp.headers = interceptedUris[url].headers;
            if (interceptedUris[url].contentType) {
                resp.headers['content-type'] =  interceptedUris[url].contentType;
            }
            resp.request = {
                href: url
            };
            setTimeout(function() {
                callback(null, resp, interceptedUris[uri].response);
            }, 2);
            return;
{'fraction_non_alphanumeric': 0.11808520301571442, 'fraction_numerical': 0.00399672994822418, 'mean_word_length': 1.9660560344827587, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2578729', 'n_tokens_mistral': 3031, 'n_tokens_neox': 2905, 'n_words': 622}