<commit_before>'use strict';

var fs = require('fs');
var ReadableStream = require('stream').Readable;

var _ = require('lodash');
var async = require('async');

var Subtext = require('subtext');
var Wreck = require('wreck');
var Boom = require('boom');

var internals = {};

internals.defaults = {
    whitelist: {
        'bz2': '425a',
        'jpg': 'ffd8',
        'mp4': '3367',
        'ogg': '4f67',
        'pdf': '2550',
        'png': '8950',
        'tar': '7573'
    },
};

internals.isSupported = function (hex, fn) {

    var whitelist = internals.defaults.whitelist;

    var supported = _.pick(whitelist, function (value) {
        return value === hex;
    });

    if (_.isEmpty(supported)) {
        return fn(new Error());
    }

    return fn();
};

internals.validateBufferData = function (buffer, fn) {

    var data = buffer.toString('hex').slice(0, 4);
    return internals.isSupported(data, fn);
};

internals.validateStreamData = function (stream, fn) {

    var done = function (err) {
        if (err) {
            return fn(err);
        }

        return fn();
    };

    var _validate = function (data) {
        return internals.validateBufferData(data, done);
    };

    stream.on('data', _validate);
};

internals.validateTemporaryFileData = function (file, fn) {

    var stream = fs.createReadStream(file);
    return internals.validateStreamData(stream, fn);
};

internals.parseFormData = function (form) {

    return function (key, fn) {
        var field = form[key];

        if (field instanceof Buffer) {
            return internals.validateBufferData(field, fn);
        }

        if (_.isObject(field) && _.has(field, 'path')) {
            return internals.validateTemporaryFileData(field.path, fn);
        }

        if (field instanceof ReadableStream) {
            return internals.validateBufferData(field._data, fn);
        }

        return fn();
    };
};

internals.parseBufferPayload = function (stream, fn) {

    var _validate = function (err, parsed) {

        if (err) {
            return fn(err);
        }

        var items = _.keys(parsed.payload);
        var iterator = internals.parseFormData(parsed.payload);

        async.each(items, iterator, fn);
    };

    var options = {
        output: 'data',
        parse: true
    };

    return Subtext.parse(stream, null, options, _validate);
};

internals.validateRawDataPayload = function (request, fn) {

    var stream = Wreck.toReadableStream(request.payload);
    stream.headers = request.headers;

    return internals.parseBufferPayload(stream, fn);
};

internals.validateRawFilePayload = function (request, fn) {

    var _read = function (err, data) {
        if (err) {
            return fn(err);
        }

        var stream = Wreck.toReadableStream(data);
        stream.headers = request.headers;

        internals.parseBufferPayload(stream, fn);
    };

    fs.readFile(request.payload.path, _read);
};

internals.validateRawStreamPayload = function (request, fn) {

    var stream = request.payload;
    stream.headers = request.headers;

    return internals.parseBufferPayload(stream, fn);
};

internals.validate = function (request, options, fn) {

    var payload = request.payload;

    if (payload instanceof Buffer) {
        return internals.validateRawDataPayload(request, fn);
    }

    if (_.isObject(payload) && _.has(payload, 'path')) {
        return internals.validateRawFilePayload(request, fn);
    }

    if (payload instanceof ReadableStream) {
        return internals.validateRawStreamPayload(request, fn);
    }

    var items = _.keys(payload);
    var iterator = internals.parseFormData(payload);

    async.each(items, iterator, fn);
};

internals.attach = function (options) {

    var method = 'post';
    var mime = 'multipart/form-data';

    return function (request, reply) {

        var done = function (err) {

            if (err) {
                return reply(Boom.unsupportedMediaType());
            }

            return reply.continue();<commit_msg>Add support for filtered whitelists.<commit_after>'use strict';

var fs = require('fs');
var ReadableStream = require('stream').Readable;

var _ = require('lodash');
var async = require('async');

var Subtext = require('subtext');
var Wreck = require('wreck');
var Boom = require('boom');

var internals = {};

internals.defaults = {
    whitelist: {
        'bz2': '425a',
        'jpg': 'ffd8',
        'mp4': '3367',
        'ogg': '4f67',
        'pdf': '2550',
        'png': '8950',
        'tar': '7573'
    },
};

internals.isSupported = function (hex, wl, fn) {

    var whitelist;
    var ref = internals.defaults.whitelist;

    if (!_.isEmpty(wl)) {
        whitelist = _.pick(ref, function (value, key) {
            return _.contains(wl, key);
        });
    } else {
        whitelist = ref;
    }

    var supported = _.pick(whitelist, function (value) {
        return value === hex;
    });

    if (_.isEmpty(supported)) {
        return fn(new Error());
    }

    return fn();
};

internals.validateBuffer = function (buffer, wl, fn) {

    var data = buffer.toString('hex').slice(0, 4);
    return internals.isSupported(data, wl, fn);
};

internals.validateStream = function (stream, wl, fn) {

    var done = function (err) {
        if (err) {
            return fn(err);
        }

        return fn();
    };

    var _validate = function (data) {
        return internals.validateBuffer(data, wl, done);
    };

    stream.on('data', _validate);
};

internals.validateTemporaryFile = function (file, wl, fn) {

    var stream = fs.createReadStream(file);
    return internals.validateStream(stream, wl, fn);
};

internals.validateFormFields = function (form, wl) {

    return function (key, fn) {
        var field = form[key];

        if (field instanceof Buffer) {
            return internals.validateBuffer(field, wl, fn);
        }

        if (_.isObject(field) && _.has(field, 'path')) {
            var file = field.path;
            return internals.validateTemporaryFile(file, wl, fn);
        }

        if (field instanceof ReadableStream) {
            var buffer = field._data;
            return internals.validateBuffer(buffer, wl, fn);
        }

        return fn();
    };
};

internals.parseBufferPayload = function (stream, wl, fn) {

    var _validate = function (err, parsed) {

        if (err) {
            return fn(err);
        }

        var payload = parsed.payload;
        var items = _.keys(payload);
        var iterator = internals.validateFormFields(payload, wl);

        async.each(items, iterator, fn);
    };

    var options = {
        output: 'data',
        parse: true
    };

    return Subtext.parse(stream, null, options, _validate);
};

internals.validateRawDataPayload = function (request, wl, fn) {

    var stream = Wreck.toReadableStream(request.payload);
    stream.headers = request.headers;

    return internals.parseBufferPayload(stream, wl, fn);
};

internals.validateRawFilePayload = function (request, wl, fn) {

    var _read = function (err, data) {
        if (err) {
            return fn(err);
        }

        var stream = Wreck.toReadableStream(data);
        stream.headers = request.headers;

        internals.parseBufferPayload(stream, wl, fn);
    };

    fs.readFile(request.payload.path, _read);
};

internals.validateRawStreamPayload = function (request, wl, fn) {

    var stream = request.payload;
    stream.headers = request.headers;

    return internals.parseBufferPayload(stream, wl, fn);
};

internals.validate = function (request, wl, fn) {

    var payload = request.payload;

    if (payload instanceof Buffer) {
        return internals.validateRawDataPayload(request, wl, fn);
    }

    if (_.isObject(payload) && _.has(payload, 'path')) {
        return internals.validateRawFilePayload(request, wl, fn);
    }

    if (payload instanceof ReadableStream) {
        return internals.validateRawStreamPayload(request, wl, fn);
    }

    var items = _.keys(payload);
    var iterator = internals.validateFormFields(payload, wl);

    async.each(items, iterator, fn);
};

internals.attach = function (options) {

    var method = 'post';
    var mime = 'multipart/form-data';

    return function (request, reply) {

        var done = function (err) {

            if (err) {
                return reply(Boom.unsupportedMediaType());
            }

            return reply.continue();
{'fraction_non_alphanumeric': 0.1330965068087626, 'fraction_numerical': 0.006394316163410302, 'mean_word_length': 2.5442719261435167, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7605959', 'n_tokens_mistral': 2615, 'n_tokens_neox': 2503, 'n_words': 608}