<commit_before>var xml2js = require("xml2js"),
    deepmerge = require("deepmerge");

var Stream = require("stream"),
    ReadableStream = Stream.Readable,
    PassThroughStream = Stream.PassThrough;

var fetch = require("./request.js"),
    parsing = require("./parse.js"),
    responseHandlers = require("./response.js");

var adapter = module.exports = {

    createReadStream: function createReadStream(url, filePath, options) {
        var outStream = new PassThroughStream();
        adapter
            .getFileStream(url, filePath, options)
            .then(function __handleStream(stream) {
                stream.pipe(outStream);
            })
            .catch(function __handleReadError(err) {
                outStream.emit("error", err);
            });
        return outStream;
    },

    getDirectoryContents: function getDirectoryContents(url, dirPath, options) {
        dirPath = dirPath || "/";
        options = deepmerge({ headers: {} }, options || {});
        var fetchURL = url + dirPath;
        return fetch(
                fetchURL,
                {
                    method: "PROPFIND",
                    headers: deepmerge(
                        {
                            Depth: 1
                        },
                        options.headers
                    )
                }
            )
            .then(responseHandlers.handleResponseCode)
            .then(function(res) {
                return res.text();
            })
            .then(function(body) {
                var parser = new xml2js.Parser({
                    ignoreAttrs: true
                });
                return new Promise(function(resolve, reject) {
                    parser.parseString(body, function (err, result) {
                        if (err) {
                            reject(err);
                        } else {
                            resolve(parsing.parseDirectoryLookup(dirPath, result));
                        }
                    });
                });
            });
    },

    getFileContents: function getFileContents(url, filePath, options) {
        options = deepmerge({ headers: {} }, options || {});
        return fetch(url + filePath, {
                method: "GET",
                headers: options.headers
            })
            .then(responseHandlers.handleResponseCode)
            .then(function(res) {
                return res.buffer();
            });
    },

    getFileStream: function getFileStream(url, filePath, options) {
        options = deepmerge({ headers: {} }, options || {});
        return fetch(url + filePath, {
                method: "GET",
                headers: deepmerge(
                    {
                        // todo range
                    },
                    options.headers
                )
            })<commit_msg>Add support for streamed ranges<commit_after>var xml2js = require("xml2js"),
    deepmerge = require("deepmerge");

var Stream = require("stream"),
    ReadableStream = Stream.Readable,
    PassThroughStream = Stream.PassThrough;

var fetch = require("./request.js"),
    parsing = require("./parse.js"),
    responseHandlers = require("./response.js");

var adapter = module.exports = {

    createReadStream: function createReadStream(url, filePath, options) {
        var outStream = new PassThroughStream();
        adapter
            .getFileStream(url, filePath, options)
            .then(function __handleStream(stream) {
                stream.pipe(outStream);
            })
            .catch(function __handleReadError(err) {
                outStream.emit("error", err);
            });
        return outStream;
    },

    getDirectoryContents: function getDirectoryContents(url, dirPath, options) {
        dirPath = dirPath || "/";
        options = deepmerge({ headers: {} }, options || {});
        var fetchURL = url + dirPath;
        return fetch(
                fetchURL,
                {
                    method: "PROPFIND",
                    headers: deepmerge(
                        {
                            Depth: 1
                        },
                        options.headers
                    )
                }
            )
            .then(responseHandlers.handleResponseCode)
            .then(function(res) {
                return res.text();
            })
            .then(function(body) {
                var parser = new xml2js.Parser({
                    ignoreAttrs: true
                });
                return new Promise(function(resolve, reject) {
                    parser.parseString(body, function (err, result) {
                        if (err) {
                            reject(err);
                        } else {
                            resolve(parsing.parseDirectoryLookup(dirPath, result));
                        }
                    });
                });
            });
    },

    getFileContents: function getFileContents(url, filePath, options) {
        options = deepmerge({ headers: {} }, options || {});
        return fetch(url + filePath, {
                method: "GET",
                headers: options.headers
            })
            .then(responseHandlers.handleResponseCode)
            .then(function(res) {
                return res.buffer();
            });
    },

    getFileStream: function getFileStream(url, filePath, options) {
        options = deepmerge({ headers: {} }, options || {});
        if (typeof options.range === "object") {
            options.headers.Range = "bytes=" + options.range.start + "-" +
                options.range.end;
        }
        return fetch(url + filePath, {
                method: "GET",
                headers: options.headers
            })
{'fraction_non_alphanumeric': 0.10920436817472699, 'fraction_numerical': 0.0013867221355520888, 'mean_word_length': 1.31355252606255, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3060970', 'n_tokens_mistral': 1410, 'n_tokens_neox': 1331, 'n_words': 275}