<commit_before>"use strict";
/**
 * Simple http client
 * make request and callback with response or error
 */
var querystring = require('querystring')
, murl  = require('url')
;
/**
 * Request object
 * Params:
 *  - httpClient: httpClient
 *  - url
 *  - method: http method
 *  - headers
 *  - params
 *  - payload
 */
function Request(httpClient, url, method, headers, params, payload) {
    url = murl.parse(url, true); // query string parsing
    this.httpClient   = httpClient;
    this.scheme       = url.protocol.substr(0, url.protocol.length - 1);
    this.port         = url.port ? url.port : (this.scheme === 'http' ? 80 : 443);
    this.host         = url.hostname;
    this.method       = method;
    this.path_info    = url.pathname;
    this.query_string = url.query || {};
    this.headers      = headers || {},
    this.params       = params || {};
    this.payload      = payload || null;
    this.headers['host'] = url.hostname;
}
Request.prototype = {
    get uri() {
        return this._formatPath(this.path_info, this.params);
    },
    /**
     * Format uri with params
     * add orphelin params in query string
     */
    _formatPath: function(path, params) {
        var queryString = this.query_string;
        for (var param in params) {
            var re = new RegExp(":"+ param)
            var found = false;
            if (path.search(re) != -1) {
                path = path.replace(re, params[param]);
                found = true;
            }
            for (var query in queryString) {
                if (queryString[query].search
                    && queryString[query].search(re) != -1) {
                    queryString[query] = queryString[query].replace(re, params[param]);
                    found = true;
                }
            }
            // exclude params in headers
            for (var header in this.headers) {
                if (this.headers[header].search(re) != -1) {
                    found = true;
                }
            }
            if (!found)
                queryString[param] = params[param];
        }
        var query = querystring.stringify(queryString);
        return path + ((query != "") ? "?"+ query : "");
    },
    /**
     * format final headers
     */
    _formatHeaders: function(headers, params) {
        for (var param in params) {
            var re = new RegExp(":"+ param)
            for (var header in headers) {
                if (headers[header].search(re) != -1) {
                    headers[header] = headers[header].replace(re, params[param]);
                }
            }
        }
        return headers;
    },
    /**
     * Return a flat object with final headers and uri
     */
    finalize: function(callback) {
        var request = {
            port   : this.port,
            host   : this.host,
            scheme : this.scheme,
            method : this.method,
            uri    : this.uri,
            headers: this._formatHeaders(this.headers, this.params),
            params : this.params,
            payload: this.payload
        };
        var client      = this.httpClient.createClient(request.port, request.host, request.scheme == 'https');
        var httpRequest = client.request(request.method, request.uri, request.headers);
        client.on('error', function(e) {
            callback(e, null);
        });
        if (request.payload !== null)
            httpRequest.write(request.payload);
        var data = "";
        httpRequest.on('response', function(response) {
            response.on('data', function(chunk) {
                data += chunk;
            });
            response.on('end', function() {
                var c = {
                    status  : response.statusCode,
                    headers : response.headers,
                    body    : data
                };
                callback(null, c);
            });
        });
        httpRequest.end();
    }
};

exports.createRequest = function(httpClient, url, method, headers, params, payload) {
    return new Request(httpClient, url, method, headers, params, payload);
};<commit_msg>refactor some stuff inside httpclient<commit_after>"use strict";
/**
 * Simple http client
 * make request and callback with response or error
 */
var querystring = require('querystring')
, murl  = require('url')
;
/**
 * Request object
 * Params:
 *  - httpClient: httpClient
 *  - url
 *  - method: http method
 *  - headers
 *  - params
 *  - payload
 */
function Request(httpClient, url, method, headers, params, payload) {
    url = murl.parse(url, true); // query string parsing
    this.httpClient   = httpClient;
    this.scheme       = url.protocol.substr(0, url.protocol.length - 1);
    this.port         = url.port ? url.port : (this.scheme === 'http' ? 80 : 443);
    this.host         = url.hostname;
    this.method       = method;
    this.path_info    = url.pathname;
    this.query_string = url.query || {};
    this.headers      = headers || {},
    this.params       = params || {};
    this.payload      = payload || null;
    this.headers['host'] = url.hostname;
}
Request.prototype = {
    get uri() {
        return this._formatPath(this.path_info, this.params);
    },
    /**
     * Format uri with params
     * add orphelin params in query string
     */
    _formatPath: function(path, params) {
        var queryString = this.query_string;
        for (var param in params) {
            var re = new RegExp(":"+ param)
            var found = false;
            if (path.search(re) != -1) {
                path = path.replace(re, params[param]);
                found = true;
            }
            for (var query in queryString) {
                if (queryString[query].search
                    && queryString[query].search(re) != -1) {
                    queryString[query] = queryString[query].replace(re, params[param]);
                    found = true;
                }
            }
            // exclude params in headers
            for (var header in this.headers) {
                if (this.headers[header].search(re) != -1) {
                    found = true;
                }
            }
            if (!found)
                queryString[param] = params[param];
        }
        var query = querystring.stringify(queryString);
        return path + ((query != "") ? "?"+ query : "");
    },
    /**
     * format final headers
     */
    _formatHeaders: function(headers, params) {
        var newHeaders = {};
        for (var header in headers)
            newHeaders[header] = headers[header];
        for (var param in params) {
            var re = new RegExp(":"+ param);
            for (var header in newHeaders) {
                if (newHeaders[header].search(re) != -1) {
                    newHeaders[header] = newHeaders[header].replace(re, params[param]);
                }
            }
        }
        return newHeaders;
    },
    /**
     * Make request to the server
     * Params:
     *  - callback
     */
    finalize: function(callback) {
        var headers     = this._formatHeaders(this.headers, this.params);
        var client      = this.httpClient.createClient(this.port, this.host, this.scheme == 'https');
        var httpRequest = client.request(this.method, this.uri, headers);
        client.on('error', function(e) {
            callback(e, null);
        });
        if (this.payload !== null)
            httpRequest.write(this.payload);
        var data = "";
        httpRequest.on('response', function(response) {
            response.on('data', function(chunk) {
                data += chunk;
            });
            response.on('end', function() {
                var c = {
                    status  : response.statusCode,
                    headers : response.headers,
                    body    : data
                };
                callback(null, c);
            });
        });
        httpRequest.end();
    }
};
/**
 * Create http request
 */
exports.createRequest = function(httpClient, url, method, headers, params, payload) {
    return new Request(httpClient, url, method, headers, params, payload);
};
{'fraction_non_alphanumeric': 0.1280353200883002, 'fraction_numerical': 0.002698062300711307, 'mean_word_length': 1.6417233560090703, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5958060', 'n_tokens_mistral': 2235, 'n_tokens_neox': 2178, 'n_words': 537}