<commit_before>var path = require("path");
var ReadableStream = require("stream").Readable;

var directoryExists = require("directory-exists").sync,
    rimraf = require("rimraf").sync,
    mkdir = require("mkdirp").sync;

var createServer = require("../resources/webdav-server.js"),
    getAdapter = require("../../source/adapter/get.js"),
    authTools = require("../../source/auth.js");

// Date example: "Mon, 10 Oct 2016 22:00:52 +0300"
var VALID_DATE = /.+?\d{4}.+?\d\d:\d\d:\d\d/;
var SERVER_URL = "http://localhost:9999";

var TEST_DIR_PARENT_PATH = path.resolve(__dirname, "../resources/webdav_testing_files/parent"),
    TEST_DIR_PATH = path.resolve(TEST_DIR_PARENT_PATH, "./child");

var DAV_USER = "test";
var DAV_PASS = "<PASSWORD>";

describe("adapter:get", function() {

    describe.skip("authenticated", function() {

        beforeEach(function(done) {
            this.server = createServer();
            this.server.startAuthenticated().then(done);
            this.options = {
                headers: {
                    authorization: authTools.generateAuthHeader(DAV_USER, DAV_PASS)
                }
            };
        });

        afterEach(function(done) {
            this.server.stop().then(done);
        });

        describe("getDirectoryContents", function() {

            it("gets all objects in directory", function() {
                return getAdapter
                    .getDirectoryContents(SERVER_URL, "/", this.options)
                    .then(function(contents) {
                        expect(contents.length).to.equal(2);
                    });
            });

        });

    });

    describe("non-authenticated", function() {

        beforeEach(function(done) {
            this.server = createServer();
            this.server.start().then(done);
        });

        afterEach(function(done) {
            this.server.stop().then(done);
        });

        describe("createReadStream", function() {

            it("streams contents of a remote file", function(done) {
                var stream = getAdapter.createReadStream(SERVER_URL, "/gem.png");
                expect(stream instanceof ReadableStream).to.be.true;
                var buffers = [];
                stream.on("data", function(d) { buffers.push(d); });
                stream.on("end", function() {
                    var final = Buffer.concat(buffers);
                    expect(final.length).to.equal(279);
                    done();
                });
            });

        });

        describe("getDirectoryContents", function() {

            it("gets all objects in directory", function() {
                return getAdapter
                    .getDirectoryContents(SERVER_URL, "/")
                    .then(function(contents) {
                        expect(contents.length).to.equal(2);
                    });
            });

            it("gets objects in correct form", function() {
                return getAdapter
                    .getDirectoryContents(SERVER_URL, "/")
                    .then(function(contents) {
                        contents.forEach(function(item) {
                            expect(item.filename.length).to.be.above(0);
                            expect(item.basename.length).to.be.above(0);
                            expect(item.filename.indexOf(item.basename)).to.be.above(-1);
                            var type = item.type;
                            expect(["file", "directory"]).to.contain(type);
                            expect(item.lastmod).to.match(VALID_DATE);
                            if (type === "file") {
                                expect(item.size).to.be.above(0);
                                expect(item.mime.length).to.be.above(0);
                            } else if (type === "directory") {
                                expect(item.size).to.equal(0);
                            }
                        });
                    });
            });

        });

        describe("getFileContents", function() {

            it("gets contents of a remote file", function() {
                return getAdapter
                    .getFileContents(SERVER_URL, "/gem.png")
                    .then(function(contents) {
                        expect(contents.length).to.equal(279);
                        expect(contents instanceof Buffer).to.be.true;
                    });
            });

        });

        describe("getFileStream", function() {

            it("streams contents of a remote file", function() {
                return getAdapter
                    .getFileStream(SERVER_URL, "/gem.png")
                    .then(function(stream) {
                        expect(stream instanceof ReadableStream).to.be.true;
                        var buffers = [];
                        return new Promise(function(resolve) {
                            stream.on("data", function(d) { buffers.push(d); });
                            stream.on('end', function() {
                                resolve(Buffer.concat(buffers));
                            });
                        });
                    })
                    .then(function(buff) {
                        expect(buff.length).to.equal(279);
                    });
            });

        });<commit_msg>Add test for ranged streams<commit_after>var path = require("path");
var ReadableStream = require("stream").Readable;

var directoryExists = require("directory-exists").sync,
    rimraf = require("rimraf").sync,
    mkdir = require("mkdirp").sync;

var createServer = require("../resources/webdav-server.js"),
    getAdapter = require("../../source/adapter/get.js"),
    authTools = require("../../source/auth.js");

// Date example: "Mon, 10 Oct 2016 22:00:52 +0300"
var VALID_DATE = /.+?\d{4}.+?\d\d:\d\d:\d\d/;
var SERVER_URL = "http://localhost:9999";

var TEST_DIR_PARENT_PATH = path.resolve(__dirname, "../resources/webdav_testing_files/parent"),
    TEST_DIR_PATH = path.resolve(TEST_DIR_PARENT_PATH, "./child");

var DAV_USER = "test";
var DAV_PASS = "<PASSWORD>";

describe("adapter:get", function() {

    describe.skip("authenticated", function() {

        beforeEach(function(done) {
            this.server = createServer();
            this.server.startAuthenticated().then(done);
            this.options = {
                headers: {
                    authorization: authTools.generateAuthHeader(DAV_USER, DAV_PASS)
                }
            };
        });

        afterEach(function(done) {
            this.server.stop().then(done);
        });

        describe("getDirectoryContents", function() {

            it("gets all objects in directory", function() {
                return getAdapter
                    .getDirectoryContents(SERVER_URL, "/", this.options)
                    .then(function(contents) {
                        expect(contents.length).to.equal(2);
                    });
            });

        });

    });

    describe("non-authenticated", function() {

        beforeEach(function(done) {
            this.server = createServer();
            this.server.start().then(done);
        });

        afterEach(function(done) {
            this.server.stop().then(done);
        });

        describe("createReadStream", function() {

            it("streams contents of a remote file", function(done) {
                var stream = getAdapter.createReadStream(SERVER_URL, "/gem.png");
                expect(stream instanceof ReadableStream).to.be.true;
                var buffers = [];
                stream.on("data", function(d) { buffers.push(d); });
                stream.on("end", function() {
                    var final = Buffer.concat(buffers);
                    expect(final.length).to.equal(279);
                    done();
                });
            });

        });

        describe("getDirectoryContents", function() {

            it("gets all objects in directory", function() {
                return getAdapter
                    .getDirectoryContents(SERVER_URL, "/")
                    .then(function(contents) {
                        expect(contents.length).to.equal(2);
                    });
            });

            it("gets objects in correct form", function() {
                return getAdapter
                    .getDirectoryContents(SERVER_URL, "/")
                    .then(function(contents) {
                        contents.forEach(function(item) {
                            expect(item.filename.length).to.be.above(0);
                            expect(item.basename.length).to.be.above(0);
                            expect(item.filename.indexOf(item.basename)).to.be.above(-1);
                            var type = item.type;
                            expect(["file", "directory"]).to.contain(type);
                            expect(item.lastmod).to.match(VALID_DATE);
                            if (type === "file") {
                                expect(item.size).to.be.above(0);
                                expect(item.mime.length).to.be.above(0);
                            } else if (type === "directory") {
                                expect(item.size).to.equal(0);
                            }
                        });
                    });
            });

        });

        describe("getFileContents", function() {

            it("gets contents of a remote file", function() {
                return getAdapter
                    .getFileContents(SERVER_URL, "/gem.png")
                    .then(function(contents) {
                        expect(contents.length).to.equal(279);
                        expect(contents instanceof Buffer).to.be.true;
                    });
            });

        });

        describe("getFileStream", function() {

            it("streams contents of a remote file", function() {
                return getAdapter
                    .getFileStream(SERVER_URL, "/gem.png")
                    .then(function(stream) {
                        expect(stream instanceof ReadableStream).to.be.true;
                        var buffers = [];
                        return new Promise(function(resolve) {
                            stream.on("data", function(d) { buffers.push(d); });
                            stream.on('end', function() {
                                resolve(Buffer.concat(buffers));
                            });
                        });
                    })
                    .then(function(buff) {
                        expect(buff.length).to.equal(279);
                    });
            });

            it("streams portions (ranges) of a remote file", function() {
                return Promise
                    .all([
                        getAdapter.getFileStream(SERVER_URL, "/gem.png", { range: { start: 0, end: 199 } }),
                        getAdapter.getFileStream(SERVER_URL, "/gem.png", { range: { start: 200, end: 278 } })
                    ])
                    .then(function(streams) {
                        var part1 = streams.shift(),
                            part2 = streams.shift();
                        var part1Buffers = [],
                            part2Buffers = [];
                        return Promise.all([
                            new Promise(function(resolve) {
                                part1.on("data", function(d) { part1Buffers.push(d); });
                                part1.on('end', function() {
                                    resolve(Buffer.concat(part1Buffers));
                                });
                            }),
                            new Promise(function(resolve) {
                                part2.on("data", function(d) { part2Buffers.push(d); });
                                part2.on('end', function() {
                                    resolve(Buffer.concat(part2Buffers));
                                });
                            })
                        ]);
                    })
                    .then(function(buffers) {
                        var part1 = buffers.shift(),
                            part2 = buffers.shift();
                        expect(part1.length).to.equal(200);
                        expect(part2.length).to.equal(79);
                    });
            });

        });
{'fraction_non_alphanumeric': 0.13123740427247077, 'fraction_numerical': 0.008625554212011286, 'mean_word_length': 1.262217359591539, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4127655', 'n_tokens_mistral': 3166, 'n_tokens_neox': 2903, 'n_words': 470}