<commit_before>var Stream = require('stream'),
    gm = require('gm'),
    sharp,
    isOperationByEngineNameAndName = {gm: {}},
    filterConstructorByOperationName = {};

['PngQuant', 'PngCrush', 'OptiPng', 'JpegTran', 'Inkscape', 'SvgFilter'].forEach(function (constructorName) {
    try {
        filterConstructorByOperationName[constructorName.toLowerCase()] = require(constructorName.toLowerCase());
    } catch (e) {
        // SvgFilter might fail because of failed contextify installation on windows.
        // Dependency chain to contextify: svgfilter --> assetgraph --> jsdom --> contextify
    }
});

Object.keys(gm.prototype).forEach(function (propertyName) {
    if (!/^_|^(?:emit|.*Listeners?|on|once|size|orientation|format|depth|color|res|filesize|identity|write|stream)$/.test(propertyName) &&
        typeof gm.prototype[propertyName] === 'function') {
        isOperationByEngineNameAndName.gm[propertyName] = true;
    }
});

try {
    sharp = require('sharp');
} catch (e) {}

if (sharp) {
    isOperationByEngineNameAndName.sharp = {};
    ['resize', 'extract', 'sequentialRead', 'crop', 'max', 'background', 'embed', 'flatten', 'rotate', 'flip', 'flop', 'withoutEnlargement', 'sharpen', 'interpolateWith', 'gamma', 'grayscale', 'greyscale', 'jpeg', 'png', 'webp', 'quality', 'progressive', 'withMetadata', 'compressionLevel'].forEach(function (sharpOperationName) {
        isOperationByEngineNameAndName.sharp[sharpOperationName] = true;
    });
}

var engineNamesByOperationName = {};

Object.keys(isOperationByEngineNameAndName).forEach(function (engineName) {
    Object.keys(isOperationByEngineNameAndName[engineName]).forEach(function (operationName) {
        (engineNamesByOperationName[operationName] = engineNamesByOperationName[operationName] || []).push(engineName);
    });
});

module.exports = function getFilterInfosAndTargetContentTypeFromQueryString(queryString, options) {
    options = options || {};
    var filters = options.filters || {},
        filterInfos = [],
        defaultEngineName = options.defaultEngineName || (sharp && 'sharp') || 'gm',
        currentEngineName,
        operations = [],
        operationNames = [],
        usedQueryStringFragments = [],
        leftOverQueryStringFragments = [],
        targetContentType;

    function flushOperations() {
        if (operations.length > 0) {
            if (currentEngineName === 'sharp') {
                var sharpOperationsForThisInstance = [].concat(operations);
                operationNames.push('sharp');
                filterInfos.push({
                    operationName: 'sharp',
                    usedQueryStringFragments: operations.map(function (operation) {
                        return operation.usedQueryStringFragment;
                    }),
                    create: function () {
                        return sharpOperationsForThisInstance.reduce(function (sharpInstance, sharpOperation) {
                            return sharpInstance[sharpOperation.name].apply(sharpInstance, sharpOperation.args)
                        }, sharp());
                    }
                });
            } else if (currentEngineName === 'gm') {
                var gmOperationsForThisInstance = [].concat(operations);
                operationNames.push('gm');
                filterInfos.push({
                    operationName: 'gm',
                    usedQueryStringFragments: operations.map(function (gmOperation) {
                        return gmOperation.usedQueryStringFragment;
                    }),
                    create: function () {
                        // For some reason the gm module doesn't expose itself as a readable/writable stream,
                        // so we need to wrap it into one:

                        var readStream = new Stream();
                        readStream.readable = true;

                        var readWriteStream = new Stream();
                        readWriteStream.readable = readWriteStream.writable = true;
                        var spawned = false;

                        readWriteStream.write = function (chunk) {
                            if (!spawned) {
                                spawned = true;
                                var seenData = false,
                                    hasEnded = false,
                                    gmInstance = gm(readStream);
                                if (options.gmMaxPixels) {
                                    gmInstance.limit('pixels', options.gmMaxPixels);
                                }
                                gmOperationsForThisInstance.reduce(function (gmInstance, gmOperation) {
                                    return gmInstance[gmOperation.name].apply(gmInstance, gmOperation.args);
                                }, gmInstance).stream(function (err, stdout, stderr) {
                                    if (err) {
                                        hasEnded = true;
                                        return readWriteStream.emit('error', err);
                                    }
                                    stdout.on('data', function (chunk) {
                                        seenData = true;
                                        readWriteStream.emit('data', chunk);
                                    }).on('end', function () {
                                        if (!hasEnded) {
                                            if (seenData) {
                                                readWriteStream.emit('end');
                                            } else {<commit_msg>Rename gmMaxPixels => maxPixels<commit_after>var Stream = require('stream'),
    gm = require('gm'),
    sharp,
    isOperationByEngineNameAndName = {gm: {}},
    filterConstructorByOperationName = {};

['PngQuant', 'PngCrush', 'OptiPng', 'JpegTran', 'Inkscape', 'SvgFilter'].forEach(function (constructorName) {
    try {
        filterConstructorByOperationName[constructorName.toLowerCase()] = require(constructorName.toLowerCase());
    } catch (e) {
        // SvgFilter might fail because of failed contextify installation on windows.
        // Dependency chain to contextify: svgfilter --> assetgraph --> jsdom --> contextify
    }
});

Object.keys(gm.prototype).forEach(function (propertyName) {
    if (!/^_|^(?:emit|.*Listeners?|on|once|size|orientation|format|depth|color|res|filesize|identity|write|stream)$/.test(propertyName) &&
        typeof gm.prototype[propertyName] === 'function') {
        isOperationByEngineNameAndName.gm[propertyName] = true;
    }
});

try {
    sharp = require('sharp');
} catch (e) {}

if (sharp) {
    isOperationByEngineNameAndName.sharp = {};
    ['resize', 'extract', 'sequentialRead', 'crop', 'max', 'background', 'embed', 'flatten', 'rotate', 'flip', 'flop', 'withoutEnlargement', 'sharpen', 'interpolateWith', 'gamma', 'grayscale', 'greyscale', 'jpeg', 'png', 'webp', 'quality', 'progressive', 'withMetadata', 'compressionLevel'].forEach(function (sharpOperationName) {
        isOperationByEngineNameAndName.sharp[sharpOperationName] = true;
    });
}

var engineNamesByOperationName = {};

Object.keys(isOperationByEngineNameAndName).forEach(function (engineName) {
    Object.keys(isOperationByEngineNameAndName[engineName]).forEach(function (operationName) {
        (engineNamesByOperationName[operationName] = engineNamesByOperationName[operationName] || []).push(engineName);
    });
});

module.exports = function getFilterInfosAndTargetContentTypeFromQueryString(queryString, options) {
    options = options || {};
    var filters = options.filters || {},
        filterInfos = [],
        defaultEngineName = options.defaultEngineName || (sharp && 'sharp') || 'gm',
        currentEngineName,
        operations = [],
        operationNames = [],
        usedQueryStringFragments = [],
        leftOverQueryStringFragments = [],
        targetContentType;

    function flushOperations() {
        if (operations.length > 0) {
            if (currentEngineName === 'sharp') {
                var sharpOperationsForThisInstance = [].concat(operations);
                operationNames.push('sharp');
                filterInfos.push({
                    operationName: 'sharp',
                    usedQueryStringFragments: operations.map(function (operation) {
                        return operation.usedQueryStringFragment;
                    }),
                    create: function () {
                        return sharpOperationsForThisInstance.reduce(function (sharpInstance, sharpOperation) {
                            return sharpInstance[sharpOperation.name].apply(sharpInstance, sharpOperation.args)
                        }, sharp());
                    }
                });
            } else if (currentEngineName === 'gm') {
                var gmOperationsForThisInstance = [].concat(operations);
                operationNames.push('gm');
                filterInfos.push({
                    operationName: 'gm',
                    usedQueryStringFragments: operations.map(function (gmOperation) {
                        return gmOperation.usedQueryStringFragment;
                    }),
                    create: function () {
                        // For some reason the gm module doesn't expose itself as a readable/writable stream,
                        // so we need to wrap it into one:

                        var readStream = new Stream();
                        readStream.readable = true;

                        var readWriteStream = new Stream();
                        readWriteStream.readable = readWriteStream.writable = true;
                        var spawned = false;

                        readWriteStream.write = function (chunk) {
                            if (!spawned) {
                                spawned = true;
                                var seenData = false,
                                    hasEnded = false,
                                    gmInstance = gm(readStream);
                                if (options.maxPixels) {
                                    gmInstance.limit('pixels', options.maxPixels);
                                }
                                gmOperationsForThisInstance.reduce(function (gmInstance, gmOperation) {
                                    return gmInstance[gmOperation.name].apply(gmInstance, gmOperation.args);
                                }, gmInstance).stream(function (err, stdout, stderr) {
                                    if (err) {
                                        hasEnded = true;
                                        return readWriteStream.emit('error', err);
                                    }
                                    stdout.on('data', function (chunk) {
                                        seenData = true;
                                        readWriteStream.emit('data', chunk);
                                    }).on('end', function () {
                                        if (!hasEnded) {
                                            if (seenData) {
                                                readWriteStream.emit('end');
                                            } else {
{'fraction_non_alphanumeric': 0.10866477272727272, 'fraction_numerical': 0.0001775568181818182, 'mean_word_length': 1.6493414863593603, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1689539', 'n_tokens_mistral': 2699, 'n_tokens_neox': 2548, 'n_words': 498}