<commit_before>'use strict';

const path = require('path');
const http = require('http');
const https = require('https');
const fsExtra = require('fs-extra');
const url = require('url');

const tarWrapper = require('../commandWrappers/tarWrapper');
const errors = require('../errors');

function validateOptions(options) {
    if (options.compression && !(options.compression in tarWrapper.compression)) {
        throw new errors.InvalidOptionsError(`Invalid compression: ${options.compression}`);
    }

    if (!options.compression) {
        options.compression = 'gzip';
    }

    if (!options.resolveUrl) {
        throw new errors.InvalidOptionsError('`resolveUrl` function must be provided');
    }

    if (!options.strict) {
        options.strict = false;
    }
}

function pull(hash, options, cacheDir) {
    let resolvedUrlPromise = options.resolveUrl(hash);
    const archivePath = path.join(cacheDir, `${hash}.tar${tarWrapper.compression[options.compression]}`);

    if (!(resolvedUrlPromise instanceof Promise)) {
        resolvedUrlPromise = Promise.resolve(resolvedUrlPromise);
    }

    return resolvedUrlPromise
        .then(resolvedUrl => {
            return new Promise((resolve, reject) => {
                const parsedUrl = url.parse(resolvedUrl);
                let transport;
                let done = false;

                if (parsedUrl.protocol === 'http:') {
                    transport = http;
                } else if (parsedUrl.protocol === 'https:') {
                    transport = https;
                } else {
                    done = true;
                    return reject(new InvalidProtocolError(
                        `http backend can't work with \`${parsedUrl.protocol}\` protocol. ` +
                        `Only \`http:\` and \`https:\` are supported`
                    ));
                }

                transport.get(parsedUrl.href, res => {
                    if (res.statusCode === 404) {
                        done = true;
                        
                        return reject(new errors.BundleNotFoundError);
                    } else if (res.statusCode !== 200) {
                        done = true;
                        
                        if (options.strict) {
                            return reject(new InvalidStatusCodeError(
                                `Request to \'${parsedUrl}\' failed. Invalid status code: \`${res.statusCode}\``
                            ));
                        }

                        return reject(new errors.BundleNotFoundError);
                    }
                    const tarWrapperToken = {};

                    tarWrapper.extractArchiveFromStream(res, {controlToken: tarWrapperToken});

                    res.on('end', () => {
                        if (done) {
                            return;
                        }

                        done = true;
                        resolve();
                    });
                    
                    res.on('error', error => {
                        if (done) {
                            return;
                        }

                        done = true;

                        if (tarWrapperToken.terminate) {
                            tarWrapperToken.terminate();
                        }

                        if (options.strict) {
                            return reject(new BundleDownloadError(error));
                        }

                        return reject(new errors.BundleNotFoundError(error));
                    })
                });
            });
        });
}

class InvalidProtocolError extends errors.VeendorError {}
class InvalidStatusCodeError extends errors.VeendorError {}
class BundleDownloadError extends errors.VeendorError {}

module.exports = {
    validateOptions,
    pull,
    push() {
        throw new errors.VeendorError('`http` backend is read-only, pushing is not implemented');
    },
    InvalidProtocolError,
    InvalidStatusCodeError,
    BundleDownloadError,
};<commit_msg>A little better streaming for http<commit_after>'use strict';

const path = require('path');
const http = require('http');
const https = require('https');
const fsExtra = require('fs-extra');
const url = require('url');

const tarWrapper = require('../commandWrappers/tarWrapper');
const errors = require('../errors');

function validateOptions(options) {
    if (options.compression && !(options.compression in tarWrapper.compression)) {
        throw new errors.InvalidOptionsError(`Invalid compression: ${options.compression}`);
    }

    if (!options.compression) {
        options.compression = 'gzip';
    }

    if (!options.resolveUrl) {
        throw new errors.InvalidOptionsError('`resolveUrl` function must be provided');
    }

    if (!options.strict) {
        options.strict = false;
    }
}

function pull(hash, options, cacheDir) {
    let resolvedUrlPromise = options.resolveUrl(hash);
    const archivePath = path.join(cacheDir, `${hash}.tar${tarWrapper.compression[options.compression]}`);

    if (!(resolvedUrlPromise instanceof Promise)) {
        resolvedUrlPromise = Promise.resolve(resolvedUrlPromise);
    }

    return resolvedUrlPromise
        .then(resolvedUrl => {
            return new Promise((resolve, reject) => {
                const parsedUrl = url.parse(resolvedUrl);
                let transport;
                let done = false;

                if (parsedUrl.protocol === 'http:') {
                    transport = http;
                } else if (parsedUrl.protocol === 'https:') {
                    transport = https;
                } else {
                    done = true;
                    return reject(new InvalidProtocolError(
                        `http backend can't work with \`${parsedUrl.protocol}\` protocol. ` +
                        `Only \`http:\` and \`https:\` are supported`
                    ));
                }

                transport.get(parsedUrl.href, res => {
                    if (res.statusCode === 404) {
                        done = true;

                        return reject(new errors.BundleNotFoundError);
                    } else if (res.statusCode !== 200) {
                        done = true;

                        if (options.strict) {
                            return reject(new InvalidStatusCodeError(
                                `Request to \'${parsedUrl}\' failed. Invalid status code: \`${res.statusCode}\``
                            ));
                        }

                        return reject(new errors.BundleNotFoundError);
                    }

                    const tarWrapperToken = {};

                    const extractPromise = new Promise((resolve, reject) => {
                        tarWrapper.extractArchiveFromStream(res, {controlToken: tarWrapperToken})
                            .then(resolve, reject);
                    });

                    const resPromise = new Promise((resolve, reject) => {
                        res.on('end', () => {
                            if (done) {
                                return;
                            }

                            done = true;
                            resolve();
                        });

                        res.on('error', error => {
                            if (done) {
                                return;
                            }

                            done = true;

                            if (tarWrapperToken.terminate) {
                                tarWrapperToken.terminate();
                            }

                            if (options.strict) {
                                return reject(new BundleDownloadError(error));
                            }

                            return reject(new errors.BundleNotFoundError(error));
                        })
                    });

                    return Promise.all([resPromise, extractPromise]).then(resolve, reject);
                });
            });
        });
}

class InvalidProtocolError extends errors.VeendorError {}
class InvalidStatusCodeError extends errors.VeendorError {}
class BundleDownloadError extends errors.VeendorError {}

module.exports = {
    validateOptions,
    pull,
    push() {
        throw new errors.VeendorError('`http` backend is read-only, pushing is not implemented');
    },
    InvalidProtocolError,
    InvalidStatusCodeError,
    BundleDownloadError,
};
{'fraction_non_alphanumeric': 0.1037359022556391, 'fraction_numerical': 0.0014097744360902255, 'mean_word_length': 1.3510080088373377, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3177366', 'n_tokens_mistral': 1990, 'n_tokens_neox': 1860, 'n_words': 431}