<commit_before>/*
 * Script Content Parser Adapter
 * @module   adapter/script
 * @author   genify(<EMAIL>)
 */
var util = require('util'),
    path = require('path'),
    uglify = require('uglify-js');
// ast node checker
var AST_CHECKER = {
    Dot:function(n){n.name=n.property;return n;},
    ObjectKeyVal:function(n){n.name=n.key;return n;},
    Var:function(n){var a = [];n.definitions.forEach(function(n){a.push(n.name);});return a;},
    VarDef:function(n){return n.name||{};},
    Function:function(n){return n.name||{};},
    If:function(n){return n.condition;},
    Defun:function(n){return n.name||{};},
    SymbolRef:function(n){return n.definition();},
    SymbolCatch:function(n){return n.definition();},
    Dft:function(n){return n;}
};
// reserved variable key
var KEY_RESERVED = ['lt','gt','amp','nbsp','quot','apos','copy','reg'];
// script parser
// map      file map, eg. {'/path/to/out.js':['/path/to/a.js','/path/to/b.js',...],...}
// code     code map, eg. {'/path/to/a.js':'var a = 111;','/path/to/b.js':'var b = 222;',...}
var Parser = module.exports =
    require('../util/klass.js').create();
var pro = Parser.extend(require('../util/event.js'));
/**
 * class initialization
 * @param  {Object} config - config parameters
 * @return {Void}
 */
pro.init = function(config){
    this._super(config);
    // overwrite uglify logger
    uglify.AST_Node.
        warn_function =
            this._warn.bind(this);
    // merge code
    config = config||{};
    this._mergeCodeAndToAST(
        config.map||{},
        config.code||{}
    );
};
/**
 * parse code, compress code, mangle variables
 * @param  {Object}  config - config object
 * @param  {Object}  config.bags        - variable name will be mangled map
 * @param  {Number}  config.level       - obfuscate level, value between 0 and 3
 * @param  {Boolean} config.compatible  - use nej compatible mode
 * @param  {Boolean} config.dropconsole - drop console statement
 * @param  {Object}  config.sourcemap   - output source map config
 * @return {Void}
 */
pro.parse = function(config){
    config = config||{};
    this._result = {};
    // mangle local variables
    this._ast.figure_out_scope();
    this._ast.compute_char_frequency();
    var bags = config.bags||{},
        except = this._getMangleExcept(bags);
    this._ast.mangle_names({
        sort:!0,
        except:except
    });
    // mangle api
    var level = parseInt(config.level,10);
    if (isNaN(level)||level<0||level>3){
        level = 3;
    }
    if (level>0){
        this._ast.compute_char_frequency();
        this._mangleAPI(level,bags,except);
    }
    this._generateOutput(config||{});
};
/**
 * dump result after uglify
 * @return {Object} parse result, eg. {bags:{},code:{},sourcemap:{}}
 */
pro.dump = function(){
    return this._result;
};
/**
 * output uglifyjs warn logger
 * @private
 * @return {Void}
 */
pro._warn = function(msg){
    this.emit('warn',{
        message:msg
    });
};
/**
 * dump mangle except variables
 * @private
 * @param  {Object} bags - mangled variables map
 * @return {Array}  variables list
 */
pro._getMangleExcept = function(bags){
    var ret = [
        '__proto__',
        '__defineGetter__',
        '__defineSetter__',
        '__lookupGetter__',
        '__lookupSetter__'
    ];
    bags = bags||{};
    Object.keys(bags).forEach(function(key){
        var value = bags[key];
        if (key===value){
            ret.push(key);
            delete bags[key];
        }
    });
    return ret;
};
/**
 * merge code and generator ast
 * @private
 * @param  {Object} map - file map
 * @param  {Object} codes - code map
 */
pro._mergeCodeAndToAST = function(map,codes){
    // merge code
    this._asts = {};
    this._code = {};
    // top level ast
    this._ast = uglify.parse(
        '',{fromString:!0}
    );
    Object.keys(map).forEach(
        function(file){
            // merge source
            var ret = [];
            map[file].forEach(function(name){
                var code = codes[name];
                if (!code){
                    this.emit('warn',{
                        data:[name],
                        message:'empty js code for file %s'
                    });
                }else{
                    ret.push(code);
                }
            },this);
            try{
                // parse ast
                var code = ret.join('\n'),
                    ast = uglify.parse(
                        code,{
                            fromString:!0,
                            filename:file
                        }
                    );
                this._code[file] = code;
                this._asts[file] = ast;
                // merge ast<commit_msg>support source code for script adapter<commit_after>/*
 * Script Content Parser Adapter
 * @module   adapter/script
 * @author   genify(<EMAIL>)
 */
var util = require('util'),
    path = require('path'),
    uglify = require('uglify-js');
// ast node checker
var AST_CHECKER = {
    Dot:function(n){n.name=n.property;return n;},
    ObjectKeyVal:function(n){n.name=n.key;return n;},
    Var:function(n){var a = [];n.definitions.forEach(function(n){a.push(n.name);});return a;},
    VarDef:function(n){return n.name||{};},
    Function:function(n){return n.name||{};},
    If:function(n){return n.condition;},
    Defun:function(n){return n.name||{};},
    SymbolRef:function(n){return n.definition();},
    SymbolCatch:function(n){return n.definition();},
    Dft:function(n){return n;}
};
// reserved variable key
var KEY_RESERVED = ['lt','gt','amp','nbsp','quot','apos','copy','reg'];
// script parser
// map      file map, eg. {'/path/to/out.js':['/path/to/a.js','/path/to/b.js',...],...}
// code     code map, eg. {'/path/to/a.js':'var a = 111;','/path/to/b.js':'var b = 222;',...}
var Parser = module.exports =
    require('../util/klass.js').create();
var pro = Parser.extend(require('../util/event.js'));
/**
 * class initialization
 * @param  {Object} config - config parameters
 * @return {Void}
 */
pro.init = function(config){
    this._super(config);
    // overwrite uglify logger
    uglify.AST_Node.
        warn_function =
            this._warn.bind(this);
    // merge code
    config = config||{};
    this._mergeCodeAndToAST(
        config.map||{},
        config.code||{}
    );
};
/**
 * parse code, compress code, mangle variables
 * @param  {Object}  config - config object
 * @param  {Object}  config.bags        - variable name will be mangled map
 * @param  {Number}  config.level       - obfuscate level, value between 0 and 3
 * @param  {Boolean} config.compatible  - use nej compatible mode
 * @param  {Boolean} config.dropconsole - drop console statement
 * @param  {Object}  config.sourcemap   - output source map config
 * @return {Void}
 */
pro.parse = function(config){
    config = config||{};
    this._result = {};
    // mangle local variables
    this._ast.figure_out_scope();
    this._ast.compute_char_frequency();
    var bags = config.bags||{},
        except = this._getMangleExcept(bags);
    this._ast.mangle_names({
        sort:!0,
        except:except
    });
    // mangle api
    var level = parseInt(config.level,10);
    if (isNaN(level)||level<0||level>3){
        level = 3;
    }
    if (level>0){
        this._ast.compute_char_frequency();
        this._mangleAPI(level,bags,except);
    }
    this._generateOutput(config||{});
};
/**
 * dump result after uglify
 * @return {Object} parse result, eg. {bags:{},code:{},sourcemap:{}}
 */
pro.dump = function(){
    return this._result;
};
/**
 * output uglifyjs warn logger
 * @private
 * @return {Void}
 */
pro._warn = function(msg){
    this.emit('warn',{
        message:msg
    });
};
/**
 * dump mangle except variables
 * @private
 * @param  {Object} bags - mangled variables map
 * @return {Array}  variables list
 */
pro._getMangleExcept = function(bags){
    var ret = [
        '__proto__',
        '__defineGetter__',
        '__defineSetter__',
        '__lookupGetter__',
        '__lookupSetter__'
    ];
    bags = bags||{};
    Object.keys(bags).forEach(function(key){
        var value = bags[key];
        if (key===value){
            ret.push(key);
            delete bags[key];
        }
    });
    return ret;
};
/**
 * merge code and generator ast
 * @private
 * @param  {Object} map - file map
 * @param  {Object} codes - code map
 */
pro._mergeCodeAndToAST = function(map,codes){
    // merge code
    this._asts = {};
    this._code = {};
    // top level ast
    this._ast = uglify.parse(
        '',{fromString:!0}
    );
    Object.keys(map).forEach(
        function(file){
            // merge source
            var ret = [];
            if (typeof map[file]==='string'){
                ret.push(map[file]);
            }else{
                map[file].forEach(function(name){
                    var code = codes[name];
                    if (!code){
                        this.emit('warn',{
                            data:[name],
                            message:'empty js code for file %s'
                        });
                    }else{
                        ret.push(code);
                    }
                },this);
            }
            try{
                // parse ast
                var code = ret.join('\n'),
                    ast = uglify.parse(
                        code,{
                            fromString:!0,
                            filename:file
                        }
                    );
                this._code[file] = code;
                this._asts[file] = ast;
                // merge ast
{'fraction_non_alphanumeric': 0.16387820845890055, 'fraction_numerical': 0.003533201704250234, 'mean_word_length': 2.0273670965712487, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5409550', 'n_tokens_mistral': 3015, 'n_tokens_neox': 2929, 'n_words': 631}