<commit_before>/*global setImmediate:true*/
// node 0.8 compat
if (typeof setImmediate === 'undefined') {
    setImmediate = process.nextTick;
}

var request = require('request'),
    _ = require('underscore'),
    passError = require('passerror'),
    httpErrors = require('httperrors');

module.exports = function () {
    return function http(assetConfig, fromUrl, cb) {
        var url = assetConfig.url;
        assetConfig.rawSrcProxy = function (cb) {
            var matchUrl = url.match(/^([^#]*)((?:#.*)?)$/),
                urlWithoutFragmentIdentifier = matchUrl[1],
                fragmentIdentifier = matchUrl[2],
                requestOptions = http.requestOptions || {},
                numRetries = requestOptions.numRetries || 0,
                retryDelayMilliseconds = requestOptions.retryDelayMilliseconds || 250;

            (function doRequest() {
                // Hack: Allows you to control the request options by setting the 'requestOptions' of the returned function to
                // an object.
                var req = request(_.defaults({
                    uri: urlWithoutFragmentIdentifier,
                    onResponse: true
                }, requestOptions), function (err, response) {
                    if (err || (response && response.statusCode >= 500 && response.statusCode < 600)) {
                        if (typeof numRetries === 'number' && numRetries > 0) {
                            numRetries -= 1;
                            return setTimeout(doRequest, retryDelayMilliseconds);
                        } else if (err) {
                            return cb(err);
                        }
                    }
                    if (response.statusCode >= 400) {
                        return cb(new (httpErrors[response.statusCode] || httpErrors.BadGateway)({
                            message: 'Got ' + response.statusCode + ' error retrieving ' + urlWithoutFragmentIdentifier,
                            url: urlWithoutFragmentIdentifier
                        }));
                    }
                    var metadata = {
                            url: req.uri.href + fragmentIdentifier // Might be different from the originally requested url if redirects were followed
                        },
                        buffers = [],
                        contentType = response.headers['content-type'];
                    if (contentType) {
                        var matchContentType = contentType.match(/^\s*([\w\-\+\.]+\/[\w\-\+\.]+)(?:\s|;|$)/i);
                        if (matchContentType) {
                            metadata.contentType = matchContentType[1].toLowerCase();
                            var matchCharset = contentType.match(/;\s*charset\s*=\s*(['"]|)\s*([\w\-]+)\s*\1(?:\s|;|$)/i);
                            if (matchCharset) {
                                metadata.encoding = matchCharset[2].toLowerCase();
                            }
                        } else {
                            console.warn('resolvers.http: Couldn\'t parse Content-Type ' + response.headers['content-type'] + ' while fetching ' + url);
                        }
                    }
                    if ('etag' in response.headers) {
                        metadata.etag = response.headers.etag;
                    }
                    if ('cache-control' in response.headers) {<commit_msg>http.js<commit_after>/*global setImmediate:true*/
// node 0.8 compat
if (typeof setImmediate === 'undefined') {
    setImmediate = process.nextTick;
}

var request = require('request'),
    _ = require('underscore'),
    httpErrors = require('httperrors');

module.exports = function () {
    return function http(assetConfig, fromUrl, cb) {
        var url = assetConfig.url;
        assetConfig.rawSrcProxy = function (cb) {
            var matchUrl = url.match(/^([^#]*)((?:#.*)?)$/),
                urlWithoutFragmentIdentifier = matchUrl[1],
                fragmentIdentifier = matchUrl[2],
                requestOptions = http.requestOptions || {},
                numRetries = requestOptions.numRetries || 0,
                retryDelayMilliseconds = requestOptions.retryDelayMilliseconds || 250;

            (function doRequest() {
                // Hack: Allows you to control the request options by setting the 'requestOptions' of the returned function to
                // an object.
                var req = request(_.defaults({
                    uri: urlWithoutFragmentIdentifier,
                    onResponse: true
                }, requestOptions), function (err, response) {
                    if (err || (response && response.statusCode >= 500 && response.statusCode < 600)) {
                        if (typeof numRetries === 'number' && numRetries > 0) {
                            numRetries -= 1;
                            return setTimeout(doRequest, retryDelayMilliseconds);
                        } else if (err) {
                            return cb(err);
                        }
                    }
                    if (response.statusCode >= 400) {
                        var ErrorConstructor = httpErrors[response.statusCode] || httpErrors.BadGateway;
                        return cb(new ErrorConstructor({
                            message: 'Got ' + response.statusCode + ' error retrieving ' + urlWithoutFragmentIdentifier,
                            url: urlWithoutFragmentIdentifier
                        }));
                    }
                    var metadata = {
                            url: req.uri.href + fragmentIdentifier // Might be different from the originally requested url if redirects were followed
                        },
                        buffers = [],
                        contentType = response.headers['content-type'];
                    if (contentType) {
                        var matchContentType = contentType.match(/^\s*([\w\-\+\.]+\/[\w\-\+\.]+)(?:\s|;|$)/i);
                        if (matchContentType) {
                            metadata.contentType = matchContentType[1].toLowerCase();
                            var matchCharset = contentType.match(/;\s*charset\s*=\s*(['"]|)\s*([\w\-]+)\s*\1(?:\s|;|$)/i);
                            if (matchCharset) {
                                metadata.encoding = matchCharset[2].toLowerCase();
                            }
                        } else {
                            console.warn('resolvers.http: Couldn\'t parse Content-Type ' + response.headers['content-type'] + ' while fetching ' + url);
                        }
                    }
                    if ('etag' in response.headers) {
                        metadata.etag = response.headers.etag;
                    }
                    if ('cache-control' in response.headers) {
{'fraction_non_alphanumeric': 0.11641528774344706, 'fraction_numerical': 0.006443110265046127, 'mean_word_length': 1.349501203990368, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4293953', 'n_tokens_mistral': 1650, 'n_tokens_neox': 1539, 'n_words': 345}