<commit_before>module.exports = function(grunt) {

    grunt.loadNpmTasks('grunt-contrib-copy');
    var Crypto = require('cryptojs').Crypto
    var File = require('grunt-usemin/lib/file');
    var Path = require('path');
    var CleanCSS = require('clean-css');
    var Uglify = require("uglify-js");

    var defaultOptions = {
        buildDir: 'build/',
        srcDir: 'src/',
        assetsHome: 'assets/',
        checkDeps: true,
        depsDir: 'vendors/',
        depsHashesFile: '.bowerhashes',
        cdn: null
    }

    var generateHash = function(path) {
        var content = path;
        grunt.log.writeln('generating hash: '+path);   
        if (grunt.file.exists(path)) {       
            if (path.split('.').pop()=='js') {
                content = grunt.file.read(path);
                var ast = Uglify.parser.parse(content); // parse code and get the initial AST
                ast = Uglify.uglify.ast_squeeze(ast); // get an AST with compression optimizations
                content = Uglify.uglify.gen_code(ast); // compressed code here
            } else {
                content = grunt.file.read(path, {encoding:null});
            }
        }
        var hash = Crypto.SHA256(content);
        grunt.log.writeln(hash);
        return hash;
    }

    var generateHashes = function(fileList, config) {
        var hashes = {};
        fileList.forEach(function (fileObj) {         
            var files = grunt.file.expand({nonull: true}, fileObj.src);
            files.forEach(function (filename) {
                grunt.log.writeln('filename: '+filename);
                var file = new File(filename);
                file.blocks.forEach(function(block) {
                    if (block.type=='js') {
                        for (var b in block.src) {
                            var path = config.srcDir+block.src[b];                          
                            if (path.indexOf(config.depsDir)==0) {
                                var hash = generateHash(path);
                                hashes[path] = hash;
                            }                  
                        }
                    }                   
                });
            });
        });
        return hashes;
    }

    // generate number between 0 and max from arbitratry string
    var stringToInt = function(str, max) {
        var hash = 0;
        for (var i = 0; i < str.length; i++) {
           hash = hash+str.charCodeAt(i);
        }
        return hash % (max+1);
    }

    // generate url relative to destfile or cdn url if cdn config is present
    var generateFinalUrl = function(url, destfile, config) {
        var dir = url.split('/').shift();
        
        if (config.cdn && config.cdn.hosts && config.cdn.hosts[dir]) {

            var hosts = config.cdn.hosts[dir];
            var cdnNum = stringToInt(url, hosts.length-1);
            url = hosts[cdnNum]+'/'+url;

        } else {

            var from = Path.dirname('/'+destfile);
            var to = '/'+url;
            url = Path.relative(from, to);    

        }
        return url;
    }

    // prepare url from file in srcfiledir, to destfile
    // put asset in assets/ folder
    // IMPORTANT: absolute url are not processing
    var processUrl = function(url, srcfiledir, destfile, config) {
        grunt.log.writeln("Original url: "+url);
        var newUrl = url;
        // procession only relatve urls
        if (/^\/|https:|http:|data:/i.test(url) === false && url != '') {
            var urlPath = Path.resolve(srcfiledir + '/' + url);

            // extract query string
            var urlPathParts = urlPath.split("?");
            var params;
            if (urlPathParts.length>1) {
                params = urlPathParts.pop();
            }
            urlPath = urlPathParts.shift();

            // extract anchors
            var urlPathParts = urlPath.split("#");
            var anchor;
            if (urlPathParts.length>1) {
                anchor = urlPathParts.pop();
            }
            urlPath = urlPathParts.shift();    

            grunt.log.writeln("Source path: "+urlPath);                               

            var ext = urlPath.split(".").pop();
            // TODO: 12 chars seems enough, but check collisions for cleanliness
            var newFilename = generateHash(urlPath).substr(0,12)+"."+ext;
            
            grunt.log.writeln("newFilename: "+newFilename);

            newUrl = config.assetsHome+newFilename;
            if (params) {
                newUrl += "?"+params;
            }<commit_msg>[#421] fix regex for grunt build<commit_after>module.exports = function(grunt) {

    grunt.loadNpmTasks('grunt-contrib-copy');
    var Crypto = require('cryptojs').Crypto
    var File = require('grunt-usemin/lib/file');
    var Path = require('path');
    var CleanCSS = require('clean-css');
    var Uglify = require("uglify-js");

    var defaultOptions = {
        buildDir: 'build/',
        srcDir: 'src/',
        assetsHome: 'assets/',
        checkDeps: true,
        depsDir: 'vendors/',
        depsHashesFile: '.bowerhashes',
        cdn: null
    }

    var generateHash = function(path) {
        var content = path;
        grunt.log.writeln('generating hash: '+path);   
        if (grunt.file.exists(path)) {       
            if (path.split('.').pop()=='js') {
                content = grunt.file.read(path);
                var ast = Uglify.parser.parse(content); // parse code and get the initial AST
                ast = Uglify.uglify.ast_squeeze(ast); // get an AST with compression optimizations
                content = Uglify.uglify.gen_code(ast); // compressed code here
            } else {
                content = grunt.file.read(path, {encoding:null});
            }
        }
        var hash = Crypto.SHA256(content);
        grunt.log.writeln(hash);
        return hash;
    }

    var generateHashes = function(fileList, config) {
        var hashes = {};
        fileList.forEach(function (fileObj) {         
            var files = grunt.file.expand({nonull: true}, fileObj.src);
            files.forEach(function (filename) {
                grunt.log.writeln('filename: '+filename);
                var file = new File(filename);
                file.blocks.forEach(function(block) {
                    if (block.type=='js') {
                        for (var b in block.src) {
                            var path = config.srcDir+block.src[b];                          
                            if (path.indexOf(config.depsDir)==0) {
                                var hash = generateHash(path);
                                hashes[path] = hash;
                            }                  
                        }
                    }                   
                });
            });
        });
        return hashes;
    }

    // generate number between 0 and max from arbitratry string
    var stringToInt = function(str, max) {
        var hash = 0;
        for (var i = 0; i < str.length; i++) {
           hash = hash+str.charCodeAt(i);
        }
        return hash % (max+1);
    }

    // generate url relative to destfile or cdn url if cdn config is present
    var generateFinalUrl = function(url, destfile, config) {
        var dir = url.split('/').shift();
        
        if (config.cdn && config.cdn.hosts && config.cdn.hosts[dir]) {

            var hosts = config.cdn.hosts[dir];
            var cdnNum = stringToInt(url, hosts.length-1);
            url = hosts[cdnNum]+'/'+url;

        } else {

            var from = Path.dirname('/'+destfile);
            var to = '/'+url;
            url = Path.relative(from, to);    

        }
        return url;
    }

    // prepare url from file in srcfiledir, to destfile
    // put asset in assets/ folder
    // IMPORTANT: absolute url are not processing
    var processUrl = function(url, srcfiledir, destfile, config) {
        grunt.log.writeln("Original url: "+url);
        var newUrl = url;
        // procession only relatve urls
        if (/^(\/|https:|http:|data:|\?)/i.test(url) === false && url != '') {
            var urlPath = Path.resolve(srcfiledir + '/' + url);

            // extract query string
            var urlPathParts = urlPath.split("?");
            var params;
            if (urlPathParts.length>1) {
                params = urlPathParts.pop();
            }
            urlPath = urlPathParts.shift();

            // extract anchors
            var urlPathParts = urlPath.split("#");
            var anchor;
            if (urlPathParts.length>1) {
                anchor = urlPathParts.pop();
            }
            urlPath = urlPathParts.shift();    

            grunt.log.writeln("Source path: "+urlPath);                               

            var ext = urlPath.split(".").pop();
            // TODO: 12 chars seems enough, but check collisions for cleanliness
            var newFilename = generateHash(urlPath).substr(0,12)+"."+ext;
            
            grunt.log.writeln("newFilename: "+newFilename);

            newUrl = config.assetsHome+newFilename;
            if (params) {
                newUrl += "?"+params;
            }
{'fraction_non_alphanumeric': 0.12402428447528187, 'fraction_numerical': 0.0037944492627927148, 'mean_word_length': 1.4778404512489927, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6396655', 'n_tokens_mistral': 2657, 'n_tokens_neox': 2503, 'n_words': 593}