<commit_before>// fad distribute tool, distribute static resources into local dist dir

// 'use strict';

var crypto = require('crypto');
var path = require('path');
var fs = require('fs');
var glob = require("glob")

var utils = require('./utils');

// TODO not full real path, like  'images/' + (flag ? 'a' : 'b') + '.gif' , need map to real path
var extra_res_map = {};
// res hash map
var hash_map = {};
// hash map of res from src reference files,
// ex. html, view template or assets config json file,
// for generating hashmap.json in HashMapGenerate mode.
var gen_hash_map = {};
// sea modules hash map
var seamods_hash_map = {};
// sea config reference file
var seamods_config_ref = [];

var config;
var realpath = utils.realpath;

exports = module.exports = function() {
    // load config
    config = utils.loadConfig();
    if(!config){
        return false;
    }

    // commmon resources distribute
    for(var src in config.src_pattern){
        var files = glob.sync(src, {cwd: realpath()})
        console.info('Distribute static resources'.underline)
        files.map(function(f){
            distRes(f, false, config.src_pattern[src]);
        })
    }

    if(config.mode === 'HashMapGenerate' && config.dist_hashmap){
        generate_hashmap();
    }

    // seajs modules distribute
    if(config.seamods_pattern && config.seamods_dir){
        console.info('\nDistribute seajs modules'.underline)
        distSeaMods(glob.sync(config.seamods_pattern, {cwd: realpath(config.seamods_dir)}))
    }

    // create syblinks of lib etc. for local test
    if(config.symlinks && config.symlinks.length){
        createSymlinks(config.symlinks)
    }
};

// common static resources distribute
function distRes(file, is_res, pattern){

    var dist_file = file;
    if(is_res){
        dist_file = copyRes(file, is_res)
        // res not exists
        if(!dist_file){
            return false;
        }
    }

    res = extractRes(dist_file, pattern);
    if(extra_res_map[file]){
        res = res.concat(extra_res_map[file])
    }
    var replace_list = [];
    res.forEach(function(r){
        if(hash_map[r]){
            replace_list.push([r, is_res && hashSuffix(r, hash_map[r])])
            return;
        }
        var dist_r;
        if(Object.keys(config.res_pattern).indexOf(extname(r)) > -1){
            dist_r = distRes(r, true)
        } else {
            dist_r = copyRes(r, file)
        }
        // res not exists
        if(!dist_r){
            return;
        }

        if(config.seamods_config  && r === config.seamods_config){
            seamods_config_ref.push(dist_file);
        } else {
            hash_map[r] = sha(dist_r)
            if(config.mode === 'HashMapGenerate' && !is_res){
                gen_hash_map[r] = hash_map[r];
            }

            var newPath = hashSuffix(r, hash_map[r])
            fs.renameSync(realpath(dist_r), realpath(path.dirname(dist_r), path.basename(newPath)));
            replace_list.push([r, is_res && newPath])
        }
    })

    if(replace_list.length && (config.mode === 'HashMapUpdate' || is_res)){
        // console.info('Update file: '.verbose + dist_file)
        updateFile(dist_file, replace_list)
    }

    return dist_file
}

// generate hash map json file for HashMapGenerate mode.
function generate_hashmap(){
    var maps = JSON.stringify(gen_hash_map, 'null', '\t');
    var hashmap_file = realpath(config.dist_dir, config.dist_hashmap);
    fs.writeFileSync(hashmap_file, maps);
    console.info('Hash map of resource was generated: ' + config.dist_hashmap)
}

// seajs modules distribute
function distSeaMods(seamods){
    seamods.map(function(f){
        distRes(f, 'seamod')
    })
    // update sea modules
    seamods.map(hashSeamod)

    // insert sea modules hash map into seajs-config.js
    if(config.seamods_config){
        var seaconf = realpath(config.dist_dir, config.seamods_config);
        var content = fs.readFileSync(seaconf, config.encoding);
        var maps = JSON.stringify(seamods_hash_map, 'null', '\t');
        content = '// sea modules hash map \nseajs.hashmap = ' + maps + '\n\n' + content;
        fs.writeFileSync(seaconf, content);
        console.info('Hash map of sea modules was writed into: ' + path.basename(seaconf))
        // rename config file with hash suffix
        hash_map[config.seamods_config] = sha(path.join(config.dist_dir, config.seamods_config));
        fs.renameSync(seaconf, hashSuffix(seaconf, hash_map[config.seamods_config]));
        // update seaconf ref files
        seamods_config_ref.forEach(function(f){
            updateFile(f, [[config.seamods_config]])
        })
    }
}

function hashSeamod(mod){
    if(mod.slice(-3) !== '.js'){
        mod += '.js'
    }
    if(seamods_hash_map[mod]){
        return;
    }<commit_msg>fix seamods_config file update logic in different mode.<commit_after>// fad distribute tool, distribute static resources into local dist dir

// 'use strict';

var crypto = require('crypto');
var path = require('path');
var fs = require('fs');
var glob = require("glob")

var utils = require('./utils');

// TODO not full real path, like  'images/' + (flag ? 'a' : 'b') + '.gif' , need map to real path
var extra_res_map = {};
// res hash map
var hash_map = {};
// hash map of res from src reference files,
// ex. html, view template or assets config json file,
// for generating hashmap.json in HashMapGenerate mode.
var gen_hash_map = {};
// sea modules hash map
var seamods_hash_map = {};
// sea config reference file
var seamods_config_ref = [];

var config;
var realpath = utils.realpath;

exports = module.exports = function() {
    // load config
    config = utils.loadConfig();
    if(!config){
        return false;
    }

    // commmon resources distribute
    for(var src in config.src_pattern){
        var files = glob.sync(src, {cwd: realpath()})
        console.info('Distribute static resources'.underline)
        files.map(function(f){
            distRes(f, false, config.src_pattern[src]);
        })
    }

    // seajs modules distribute
    if(config.seamods_pattern && config.seamods_dir){
        console.info('\nDistribute seajs modules'.underline)
        distSeaMods(glob.sync(config.seamods_pattern, {cwd: realpath(config.seamods_dir)}))
    }

    // generate hashmap when HashMapGenerate mode
    if(config.mode === 'HashMapGenerate' && config.dist_hashmap){
        generate_hashmap();
    }

    // create syblinks of lib etc. for local test
    if(config.symlinks && config.symlinks.length){
        createSymlinks(config.symlinks)
    }
};

// common static resources distribute
function distRes(file, is_res, pattern){

    var dist_file = file;
    if(is_res){
        dist_file = copyRes(file, is_res)
        // res not exists
        if(!dist_file){
            return false;
        }
    }

    res = extractRes(dist_file, pattern);
    if(extra_res_map[file]){
        res = res.concat(extra_res_map[file])
    }
    var replace_list = [];
    res.forEach(function(r){
        if(hash_map[r]){
            replace_list.push([r, is_res && hashSuffix(r, hash_map[r])])
            return;
        }
        var dist_r;
        if(Object.keys(config.res_pattern).indexOf(extname(r)) > -1){
            dist_r = distRes(r, true)
        } else {
            dist_r = copyRes(r, file)
        }
        // res not exists
        if(!dist_r){
            return;
        }

        if(config.seamods_config  && r === config.seamods_config){
            seamods_config_ref.push(dist_file);
        } else {
            hash_map[r] = sha(dist_r)
            if(config.mode === 'HashMapGenerate' && !is_res){
                gen_hash_map[r] = hash_map[r];
            }

            var newPath = hashSuffix(r, hash_map[r])
            fs.renameSync(realpath(dist_r), realpath(path.dirname(dist_r), path.basename(newPath)));
            replace_list.push([r, is_res && newPath])
        }
    })

    if(replace_list.length && (config.mode === 'HashMapUpdate' || is_res)){
        // console.info('Update file: '.verbose + dist_file)
        updateFile(dist_file, replace_list)
    }

    return dist_file
}

// generate hash map json file for HashMapGenerate mode.
function generate_hashmap(){
    var maps = JSON.stringify(gen_hash_map, 'null', '\t');
    var hashmap_file = realpath(config.dist_dir, config.dist_hashmap);
    fs.writeFileSync(hashmap_file, maps);
    console.info('Hash map of resource was generated: ' + config.dist_hashmap)
}

// seajs modules distribute
function distSeaMods(seamods){
    seamods.map(function(f){
        distRes(f, 'seamod')
    })
    // update sea modules
    seamods.map(hashSeamod)

    // insert sea modules hash map into seajs-config.js
    if(config.seamods_config){
        var seaconf = realpath(config.dist_dir, config.seamods_config);
        var content = fs.readFileSync(seaconf, config.encoding);
        var maps = JSON.stringify(seamods_hash_map, 'null', '\t');
        content = '// sea modules hash map \nseajs.hashmap = ' + maps + '\n\n' + content;
        fs.writeFileSync(seaconf, content);
        console.info('Hash map of sea modules was writed into: ' + path.basename(seaconf))
        // rename config file with hash suffix
        hash_map[config.seamods_config] = sha(path.join(config.dist_dir, config.seamods_config));
        fs.renameSync(seaconf, hashSuffix(seaconf, hash_map[config.seamods_config]));
        // update seaconf ref files
        if(config.mode === 'HashMapUpdate'){
            seamods_config_ref.forEach(function(f){
                updateFile(f, [[config.seamods_config]])
            })
        } else {
            gen_hash_map[config.seamods_config] = hash_map[config.seamods_config];
        }
    }
}

function hashSeamod(mod){
    if(mod.slice(-3) !== '.js'){
        mod += '.js'
    }
    if(seamods_hash_map[mod]){
        return;
    }
{'fraction_non_alphanumeric': 0.1303732173561242, 'fraction_numerical': 0.00040457165975523417, 'mean_word_length': 2.6061269146608317, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3974112', 'n_tokens_mistral': 3191, 'n_tokens_neox': 3142, 'n_words': 701}