<commit_before>'use strict';

module.exports = function(grunt) {

    var fs = require("fs")

    var deleteFolderRecursive = function(path) {
        if( fs.existsSync(path) ) {
            fs.readdirSync(path).forEach(function(file,index){
                var curPath = path + "/" + file;
                if(fs.statSync(curPath).isDirectory()) { // recurse
                    deleteFolderRecursive(curPath);
                } else { // delete file
                    fs.unlinkSync(curPath);
                }
            });
            fs.rmdirSync(path);
        }
    };


    function copy_recusive( source, pattern, output ){
        var files = grunt.file.expand({}, [source+pattern]);
        for( var n in files ){
            var f = files[n].substring(source.length)
            if( f != "" ){
                if( grunt.file.isDir(files[n]) ){
                    grunt.file.mkdir( output+"/"+f )
                }else{
                    var output_f = (output+"/"+f).replace("//","/")
                    grunt.file.copy(source+"/"+f, output_f)
                }
            }
        }
    }

    grunt.registerTask("phantomizer-build", "", function () {

        var options = this.options()

        var urls = options.urls
        var build_target = options.build_target
        var clean_dir = options.clean_dir

        for( var n in clean_dir ){
            deleteFolderRecursive(clean_dir[n])
            grunt.file.mkdir(clean_dir[n])
        }

        var tasks = []
        for( var n in urls ){
            tasks.push("phantomizer-html-jitbuild:"+build_target+":"+urls[n])
        }
        grunt.task.run( tasks );

    });

    grunt.registerTask("phantomizer-export-build", "", function () {
        var done = this.async();

        var options = this.options()

        var paths = options.paths
        var build_dir = options.out_dir
        var export_dir = options.export_dir
        var rm_dir = options.rm_dir
        var rm_files = options.rm_files

        grunt.file.mkdir( export_dir )

        for( var n in paths ){
            copy_recusive(paths[n], "**/*.html", export_dir)
            copy_recusive(paths[n], "**/*.htm", export_dir)

            copy_recusive(paths[n], "**/*.js", export_dir)
            copy_recusive(paths[n], "**/*.css", export_dir)

            copy_recusive(paths[n], "**/*.jpeg", export_dir)
            copy_recusive(paths[n], "**/*.jpg", export_dir)
            copy_recusive(paths[n], "**/*.png", export_dir)
            copy_recusive(paths[n], "**/*.gif", export_dir)

            copy_recusive(paths[n], "**/*.pdf", export_dir)

            copy_recusive(paths[n], "**/*.xml", export_dir)
            copy_recusive(paths[n], "**/*.json", export_dir)
            copy_recusive(paths[n], "**/*.jsonp", export_dir)
        }


        for( var n in rm_dir ){
            deleteFolderRecursive(rm_dir[n])
        }
        for( var n in rm_files ){
            if( fs.existsSync(rm_files[n]) )
                fs.unlinkSync(rm_files[n])
        }

        done();
    });

};<commit_msg>commit<commit_after>'use strict';

module.exports = function(grunt) {

    var fs = require("fs")

    var deleteFolderRecursive = function(path) {
        if( fs.existsSync(path) ) {
            fs.readdirSync(path).forEach(function(file,index){
                var curPath = path + "/" + file;
                if(fs.statSync(curPath).isDirectory()) { // recurse
                    deleteFolderRecursive(curPath);
                } else { // delete file
                    fs.unlinkSync(curPath);
                }
            });
            fs.rmdirSync(path);
        }
    };


    function copy_recusive( source, pattern, output ){
        var files = grunt.file.expand({}, [source+pattern]);
        for( var n in files ){
            var f = files[n].substring(source.length)
            if( f != "" ){
                if( grunt.file.isDir(files[n]) ){
                    grunt.file.mkdir( output+"/"+f )
                }else{
                    var output_f = (output+"/"+f).replace("//","/")
                    grunt.file.copy(source+"/"+f, output_f)
                }
            }
        }
    }

    grunt.registerTask("phantomizer-build", "", function () {

        var options = this.options();
        var grunt_config = grunt.config.get();

        var build_target = options.build_target;
        var clean_dir = options.clean_dir;

        for( var n in clean_dir ){
            deleteFolderRecursive(clean_dir[n])
            grunt.file.mkdir(clean_dir[n])
        }

        var tasks = [];
        if( options.urls ){
            for( var n in options.urls ){
                tasks.push("phantomizer-html-jitbuild:"+build_target+":"+options.urls[n])
            }
        }else if (grunt_config.routing){
            for( var n in grunt_config.routing ){
                var route = grunt_config.routing[n];
                var url = route.template;
                tasks.push("phantomizer-html-jitbuild:"+build_target+":"+url)
            }
        }
        grunt.task.run( tasks );

    });

    grunt.registerMultiTask("phantomizer-export-build", "", function () {
        var done = this.async();

        var options = this.options();

        var paths = options.paths;
        var copy_patterns = options.copy_patterns;
        var export_dir = options.export_dir;
        var rm_dir = options.rm_dir;
        var rm_files = options.rm_files;

        grunt.log.ok("exporting to "+export_dir );
        grunt.file.mkdir( export_dir );

        for( var n in paths ){
            for( var k in copy_patterns ){
                grunt.log.ok("copying "+paths[n]+""+copy_patterns[n]);
                copy_recusive(paths[n], copy_patterns[k], export_dir);
            }
        }


        for( var n in rm_dir ){
            grunt.log.ok("deleting "+rm_dir[n]);
            deleteFolderRecursive(rm_dir[n]);
        }
        for( var n in rm_files ){
            grunt.log.ok("deleting "+rm_files[n]);
            if( fs.existsSync(rm_files[n]) )
                fs.unlinkSync(rm_files[n]);
        }

        done();
    });

};
{'fraction_non_alphanumeric': 0.14218086842964414, 'fraction_numerical': 0.0, 'mean_word_length': 1.6767147225862822, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2171421', 'n_tokens_mistral': 1913, 'n_tokens_neox': 1833, 'n_words': 330}