<commit_before>/*jslint node: true*/

"use strict";

var util = require("util"),
    q = require("q"),
    request = require("request"),
    HTTP_STATUSES = require("./httpStatuses"),
    limit = require("./limit");

module.exports = (function urlReader() {
    var TIME = {ONE_SECOND: 1000},

        readUri = limit(
            q.denodeify(request),
            TIME.ONE_SECOND * 10
        ),

        readUrl = function readUrl(url, refererUrl) {
            return readUri({
                url: url,
                headers: {
                    "Referer": refererUrl,
                    //pretend to be something that doesn't support plugins
                    "User-Agent": "Mozilla/5.0 (Linux; Android 4.3; Nexus 7 Build/JSS15Q) " +
                        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.72 Safari/537.36"
                },
                gzip: true,
                timeout: TIME.ONE_SECOND * 30,
                maxRedirects: 4
            }).
                then(function (args) {
                    var response = args[0],
                        statusCode = response.statusCode,
                        body = args[1];
                    if (statusCode !== HTTP_STATUSES.OK) {
                        throw {
                            name: "unexpected status",
                            message: util.format(
                                "expected status to be %d (%s)",
                                HTTP_STATUSES.OK,
                                HTTP_STATUSES[HTTP_STATUSES.OK]
                            ),
                            url: url,
                            expectedStatusCode: HTTP_STATUSES.OK,
                            expectedStatusDescription: HTTP_STATUSES[HTTP_STATUSES.OK],
                            statusCode: statusCode,
                            statusDescription: HTTP_STATUSES[statusCode]
                        };
                    }
                    return body;
                });
        },

        initialise = function initialise() {
            return {readUrl: readUrl};
        };

    return initialise();
}());
<commit_msg>Moved creation of error object into own function<commit_after>/*jslint node: true*/

"use strict";

var util = require("util"),
    q = require("q"),
    request = require("request"),
    HTTP_STATUSES = require("./httpStatuses"),
    limit = require("./limit"),

    unexpectedStatus = function unexpectedStatus(url, expectedStatusCode, statusCode) {
        return {
            name: "unexpected status",
            message: util.format(
                "expected status to be %d (%s)",
                expectedStatusCode,
                HTTP_STATUSES[expectedStatusCode]
            ),
            url: url,
            expectedStatusCode: expectedStatusCode,
            expectedStatusDescription: HTTP_STATUSES[expectedStatusCode],
            statusCode: statusCode,
            statusDescription: HTTP_STATUSES[statusCode]
        };
    };

module.exports = (function urlReader() {
    var TIME = {ONE_SECOND: 1000},

        readUri = limit(
            q.denodeify(request),
            TIME.ONE_SECOND * 10
        ),

        readUrl = function readUrl(url, refererUrl) {
            return readUri({
                url: url,
                headers: {
                    "Referer": refererUrl,
                    //pretend to be something that doesn't support plugins
                    "User-Agent": "Mozilla/5.0 (Linux; Android 4.3; Nexus 7 Build/JSS15Q) " +
                        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.72 Safari/537.36"
                },
                gzip: true,
                timeout: TIME.ONE_SECOND * 30,
                maxRedirects: 4
            }).
                then(function (args) {
                    var response = args[0],
                        statusCode = response.statusCode,
                        body = args[1];
                    if (statusCode !== HTTP_STATUSES.OK) {
                        throw unexpectedStatus(url, HTTP_STATUSES.OK, statusCode);
                    }
                    return body;
                });
        },

        initialise = function initialise() {
            return {readUrl: readUrl};
        };

    return initialise();
}());

{'fraction_non_alphanumeric': 0.09993028119916338, 'fraction_numerical': 0.017197304206367652, 'mean_word_length': 1.2652631578947369, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3066879', 'n_tokens_mistral': 1170, 'n_tokens_neox': 1056, 'n_words': 244}