<commit_before>var config = './notes/example-patterns-config';

var fs = require( 'fs' )
    , util = require( 'util' )
    , configFile = require( config)
    , css = require( 'css' )
    , example = require( './notes/example-config')
    , async = require( 'async' )
    , commentParser = require( 'comment-parser' )
    ;
// run in the terminal using `node index.js`
module.exports = {

    configObj: {},

    config: require( './notes/parseFiles/input' ).patterns.sections,
    
    init: function( options ) {
        var configData;
        
        this.readFile();
        // console.log(configData);
        // this.getFiles();
        //this.writeFile();
    },
    getFiles: function() {
        var files;
       
        var glob = require("glob");

        var key = Object.keys(configFile);
        // console.log(configFile[key].sections.length);

        for ( var i = 0, l = configFile[key].sections.length; i < l; i++ ) {
            var sectObjs = configFile[key].sections[i];
            var ObjFiles = sectObjs.files;
            // console.log('file name ', ObjFiles);

            if ( ObjFiles.indexOf( '*' ) > -1 ) {

                files = glob.sync( ObjFiles, { nodir: true, matchBase:true } );
                console.log(util.inspect(files, { depth:5, colors: true }));
            }
            else {
                var filesStat = fs.statSync( ObjFiles );
                var filesTest = filesStat.isFile();
                // console.log(filesTest);
                if ( filesTest === false ) {
                    files = glob.sync( ObjFiles+'*', { nodir: true, matchBase:true } );
                    console.log(util.inspect(files, { depth:5, colors: true }));
                }
            }
        }
    },
    readFile: function() {
        //get the data and throw it into a variable that we can use to edit
        this.configObj = fs.readFileSync('./notes/example-patterns-config.js', 'utf-8');
        console.log(this.configObj);
        files = glob.sync( example.patterns.sections[1].files, { nodir: true } );
        
        this.parseFiles();
    },
    parseFiles: function() {
        
        var sections = this.config;
        
        async.each( sections, this.parseFile, function() {
            
            console.log( 'DONE!', util.inspect( sections, { depth:6, colors:true } ));
        });
    },
    parseFile: function( section, callback) {
        
        //internal function that parses using comment-parse on currentFile
        function parseComment( currentFile, data ) {
            
            // grab each comment block
            var comments = data.split( '/**' );
            
            // the first array item is always an empty string, so we shift
            comments.shift();
            
            
            for ( var i = 0; i < comments.length; i++ ) {
                
                // split blocks into comment and code content
                var block = comments[ i ].split(/^\s*\*\//m)
                    , toParse = '/**' + block[ 0 ] + ' */'<commit_msg>index updates<commit_after>var configLocation = './notes/example-patterns-config';

var fs = require( 'fs' )
    , util = require( 'util' )
    , configFile = require( configLocation )
    , css = require( 'css' )
    , example = require( './notes/example-config')
    , async = require( 'async' )
    , commentParser = require( 'comment-parser' )
    ;
// run in the terminal using `node index.js`
module.exports = {

    configObj: {},

    config: require( './notes/parseFiles/input' ).patterns.sections,
    
    init: function( options ) {

        // var configData = this.configObj;
        
        this.readFile();
        console.log(this.configObj);
        this.getFiles(this.configObj);
        //this.writeFile();
    },
    getFiles: function( data ) {

        var configData = data
            , files;
       
        var glob = require("glob");

        console.log(configData.config);
        // var key = Object.keys( configData );
        // // console.log(configFile[key].sections.length);

        // for ( var i = 0, l = data[key].sections.length; i < l; i++ ) {
        //     var sectObjs = data[key].sections[i];
        //     var ObjFiles = sectObjs.files;
        //     console.log('file name ', ObjFiles);

        //     if ( ObjFiles.indexOf( '*' ) > -1 ) {

        //         files = glob.sync( ObjFiles, { nodir: true, matchBase:true } );
        //         console.log(util.inspect(files, { depth:5, colors: true }));
        //     }
        //     else {
        //         var filesStat = fs.statSync( ObjFiles );
        //         var filesTest = filesStat.isFile();
        //         // console.log(filesTest);
        //         if ( filesTest === false ) {
        //             files = glob.sync( ObjFiles+'*', { nodir: true, matchBase:true } );
        //             console.log(util.inspect(files, { depth:5, colors: true }));
        //         }
        //     }
        // }
    },
    readFile: function() {
        //get the data and throw it into a variable that we can use to edit
        this.configObj = fs.readFileSync('./notes/example-patterns-config.js', 'utf-8');
        // console.log(this.configObj);
        
        // this.parseFiles();
    },
    parseFiles: function() {
        
        var sections = this.config;
        
        async.each( sections, this.parseFile, function() {
            
            // console.log( 'DONE!', util.inspect( sections, { depth:6, colors:true } ));
        });
    },
    parseFile: function( section, callback) {
        
        //internal function that parses using comment-parse on currentFile
        function parseComment( currentFile, data ) {
            
            // grab each comment block
            var comments = data.split( '/**' );
            
            // the first array item is always an empty string, so we shift
            comments.shift();
            
            
            for ( var i = 0; i < comments.length; i++ ) {
                
                // split blocks into comment and code content
                var block = comments[ i ].split(/^\s*\*\//m)
                    , toParse = '/**' + block[ 0 ] + ' */'
{'fraction_non_alphanumeric': 0.14157158926562752, 'fraction_numerical': 0.0027318013819701108, 'mean_word_length': 1.5665979381443298, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3118555', 'n_tokens_mistral': 1770, 'n_tokens_neox': 1738, 'n_words': 424}