<commit_before>/**
 * The file cache class.
 *
 * @class Cache
 */
var fs = require('fs');
var zlib = require('zlib');
var mime = require('mime');
var crypto = require('crypto');
var minifyHtml = require('html-minifier').minify;
var minifyCSS =  require('clean-css');
var UglifyJS = require('uglify-js');
var html_minifier_options = {
    removeComments: true,
    removeCommentsFromCDATA: true,
    removeCDATASectionsFromCDATA: true,
    removeAttributeQuotes: true,
    removeRedundantAttributes: true,
    removeEmptyAttributes: true,
    removeOptionalTags: true,
    collapseBooleanAttributes: true,
    collapseWhitespace: true
};

// regular expressions
//var removeFingerprint = /\.[^\.]+\.js$/;
var suffix = /\.\w+$/;
var checkFp = /\.@[a-z0-9]{7}\.(?:js|css|html)$/i;

// global file cache
var fileCache = engine.pojo('files');

// this is not quaranteed to be stable! why?
zlib.Z_DEFAULT_COMPRESSION = 9;

/**
 * Read a file or get it from the cache.
 *
 * @public
 * @param {string} The file path.
 * @param {function} The callback function.
 * @param {array} Give an array with allowed paths. (optional)
 * @param {string} A string, which is prepended to a path.
 */
exports.get = getFile;
function getFile (parentDir, path, callback, options) {

    // path must be absolute
    if (path.indexOf('../') !== -1) {
        var err = new Error('Invalid path.');
        console.error('lib/file.js#80', err.toString());
        return callback(err);
    }

    options = options || {};

    // set default working dir to public app folder
    parentDir = parentDir || engine.paths.app_public;

    // check if the file is allowed
    if (options.allowedFiles) {

        var found = false;

        // try to find the file
        for (var i = 0; i < options.allowedFiles.length; ++i) {
            if (options.allowedFiles[i].replace(/^\.\//, '') === path) {
                found = true;
                break;
            }
        }

        // return not found if requested file is not found in "allowedFiles"
        if (!found) {
            var err = new Error('File not found.');
            console.error('lib/file.js#158', err.toString());
            return callback(err);
        }
    }

    // create a clean path without the fingerprint
    var cleanPath = removeFingerprint(path);

    // check if path had fingerprint
    var hasFingerprint = cleanPath === path ? false : true;

    // check if file is already in cache
    var file = fileCache.get(cleanPath);

    if (file) {
        return callback(null, file, true);
    }

    // TODO this looks a little whacky.. find a better way to distinct core client and module client files
    var readPath = parentDir + (options.allowedFiles ? (options.prependPath || '') : '') + (cleanPath[0] === '/' ? cleanPath.substr(1) : cleanPath);

    readFile(file, parentDir, readPath, cleanPath, options, hasFingerprint, callback);
};

function readFile (file, parentDir, path, cleanPath, options, hasFingerprint, callback) {

    var mimeType = mime.lookup(path);
    fs.readFile(path, function (err, data) {

        if (err) {
            console.error('lib/file.js#122', err.toString());
            return callback(err);
        }

        // minify js, html or css
        switch (mimeType) {
            case 'text/html':
                data = minifyHtml(data.toString(), html_minifier_options);
                break;
            case 'text/css':
                data = new minifyCSS({processImport: false}).minify(data.toString()).styles;
                break;
            case 'application/javascript':

                // wrap javascript files, except the engine client
                if (cleanPath !== 'engine.js') {
                    data = "E('" + (options.prependPath || '') + cleanPath + "',function(require,module,exports){\n" + data.toString() + "\nreturn module});";
                }

                if (engine.production && (data = UglifyJS.minify(data.toString(), {fromString: true}))) {
                    data = data.code;
                }
                break;
        }

        // create fingerprint
        var fingerprint = crypto.createHash('md5').update(data).digest('hex').substr(7);

        // don't compress data
        if (options.noCompression) {<commit_msg>add engine and global to the module scope<commit_after>/**
 * The file cache class.
 *
 * @class Cache
 */
var fs = require('fs');
var zlib = require('zlib');
var mime = require('mime');
var crypto = require('crypto');
var minifyHtml = require('html-minifier').minify;
var minifyCSS =  require('clean-css');
var UglifyJS = require('uglify-js');
var html_minifier_options = {
    removeComments: true,
    removeCommentsFromCDATA: true,
    removeCDATASectionsFromCDATA: true,
    removeAttributeQuotes: true,
    removeRedundantAttributes: true,
    removeEmptyAttributes: true,
    removeOptionalTags: true,
    collapseBooleanAttributes: true,
    collapseWhitespace: true
};

// regular expressions
//var removeFingerprint = /\.[^\.]+\.js$/;
var suffix = /\.\w+$/;
var checkFp = /\.@[a-z0-9]{7}\.(?:js|css|html)$/i;

// global file cache
var fileCache = engine.pojo('files');

// this is not quaranteed to be stable! why?
zlib.Z_DEFAULT_COMPRESSION = 9;

/**
 * Read a file or get it from the cache.
 *
 * @public
 * @param {string} The file path.
 * @param {function} The callback function.
 * @param {array} Give an array with allowed paths. (optional)
 * @param {string} A string, which is prepended to a path.
 */
exports.get = getFile;
function getFile (parentDir, path, callback, options) {

    // path must be absolute
    if (path.indexOf('../') !== -1) {
        var err = new Error('Invalid path.');
        console.error('lib/file.js#80', err.toString());
        return callback(err);
    }

    options = options || {};

    // set default working dir to public app folder
    parentDir = parentDir || engine.paths.app_public;

    // check if the file is allowed
    if (options.allowedFiles) {

        var found = false;

        // try to find the file
        for (var i = 0; i < options.allowedFiles.length; ++i) {
            if (options.allowedFiles[i].replace(/^\.\//, '') === path) {
                found = true;
                break;
            }
        }

        // return not found if requested file is not found in "allowedFiles"
        if (!found) {
            var err = new Error('File not found.');
            console.error('lib/file.js#158', err.toString());
            return callback(err);
        }
    }

    // create a clean path without the fingerprint
    var cleanPath = removeFingerprint(path);

    // check if path had fingerprint
    var hasFingerprint = cleanPath === path ? false : true;

    // check if file is already in cache
    var file = fileCache.get(cleanPath);

    if (file) {
        return callback(null, file, true);
    }

    // TODO this looks a little whacky.. find a better way to distinct core client and module client files
    var readPath = parentDir + (options.allowedFiles ? (options.prependPath || '') : '') + (cleanPath[0] === '/' ? cleanPath.substr(1) : cleanPath);

    readFile(file, parentDir, readPath, cleanPath, options, hasFingerprint, callback);
};

function readFile (file, parentDir, path, cleanPath, options, hasFingerprint, callback) {

    var mimeType = mime.lookup(path);
    fs.readFile(path, function (err, data) {

        if (err) {
            console.error('lib/file.js#122', err.toString());
            return callback(err);
        }

        // minify js, html or css
        switch (mimeType) {
            case 'text/html':
                data = minifyHtml(data.toString(), html_minifier_options);
                break;
            case 'text/css':
                data = new minifyCSS({processImport: false}).minify(data.toString()).styles;
                break;
            case 'application/javascript':

                // wrap javascript files, except the engine client
                if (cleanPath !== 'engine.js') {
                    data = "E('" + (options.prependPath || '') + cleanPath + "',function(require,module,exports,global,engine){\n" + data.toString() + "\nreturn module});";
                }

                if (engine.production && (data = UglifyJS.minify(data.toString(), {fromString: true}))) {
                    data = data.code;
                }
                break;
        }

        // create fingerprint
        var fingerprint = crypto.createHash('md5').update(data).digest('hex').substr(7);

        // don't compress data
        if (options.noCompression) {
{'fraction_non_alphanumeric': 0.13713620488940628, 'fraction_numerical': 0.004190919674039581, 'mean_word_length': 2.348012470771629, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '45261', 'n_tokens_mistral': 2544, 'n_tokens_neox': 2423, 'n_words': 715}