<commit_before>'use strict';
module.exports = function () {
    return function (req, res, next) {
        // TODO: urlext parser needs to be rewritten
        var query, replaced = false;
        if (req.url.match(/\?/i)) {
            query = req.url.split('?').pop();
        }
        if (req.url.match(/\.(msgpack)/i)) {
            replaced = true;
            req.url = req.url.replace(/\.msgpack(\?.*)?$/i, '');
            req.headers['content-type'] = 'application/msgpack';
            req.headers.accept = 'application/msgpack';
        } else if (req.url.match(/\.(xml)/i)) {
            replaced = true;
            req.url = req.url.replace(/\.xml(\?.*)?$/i, '');
            req.headers['content-type'] = 'application/xml';
            req.headers.accept = 'application/xml';
        } else if (req.url.match(/\.(json)/i)) {
            replaced = true;
            req.url = req.url.replace(/\.json(\?.*)?$/i, '');
            req.headers['content-type'] = 'application/json';
            req.headers.accept = 'application/json';
        } else if (req.url.match(/\.(csv)/i)) {
            replaced = true;
            req.url = req.url.replace(/\.csv(\?.*)?$/i, '');
            req.headers['content-type'] = 'text/csv';
            req.headers.accept = 'text/csv';
        } else if (req.url.match(/\.[a-z]+(\?.*)?$/i)) {
            replaced = true;
            req.url = req.url.replace(/\.[a-z]+(\?.*)?$/i, '');
            if (!req.contentType()) {
                req.headers['content-type'] = 'application/octet-stream';
                req.headers.accept = 'application/octet-stream';
            }
        }
        if (query && replaced) {
            req.url += '?' + query;
        }
        res.setHeader('content-type', req.contentType());
        return next();
    };
};<commit_msg>rewrote urlext parser<commit_after>'use strict';
module.exports = function () {
    return function (req, res, next) {
        var url = require('url').parse(req.url),
            extension = require('path').extname(url.pathname),
            splitPath = url.pathname.split('.');

        if (splitPath.length > 1) {
            splitPath.pop();
        }

        url.pathname = splitPath.join('.');

        if (extension && extension.length > 0) {
            switch (extension.toLowerCase()) {
                case '.json':
                    req.headers['content-type'] = 'application/json';
                    req.headers.accept = 'application/json';
                    break;
                case '.jpg':
                    req.headers['content-type'] = 'image/jpeg';
                    req.headers.accept = 'image/jpeg';
                    break;
                case '.png':
                    req.headers['content-type'] = 'image/png';
                    req.headers.accept = 'image/png';
                    break;
                default:
                    req.headers['content-type'] = 'application/octet-stream';
                    req.headers.accept = 'application/octet-stream';
            }
        }
        req.url = require('url').format(url);

        res.setHeader('content-type', req.contentType());
        return next();
    };
};
{'fraction_non_alphanumeric': 0.18190386427898209, 'fraction_numerical': 0.0006283380458686773, 'mean_word_length': 1.5431309904153354, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7122154', 'n_tokens_mistral': 931, 'n_tokens_neox': 907, 'n_words': 151}