<commit_before>var configLocation = './notes/example-documentation-config';

var fs = require( 'fs' )
    , util = require( 'util' )
    , configFile = require( configLocation )
    , css = require( 'css' )
    , example = require( './notes/example-config')
    , async = require( 'async' )
    , commentParser = require( 'comment-parser' )
    ;
// run in the terminal using `node index.js`
module.exports = {

    configObj: {},

    config: require( './notes/parseFiles/input' ).documentation.sections,
    
    init: function( options ) {
        
        this.readFile();
        
        this.getFiles(this.configObj);
        
    },
    getFiles: function( data ) {

        var configData = data
            , files;
       
        var glob = require("glob");

        var key = Object.keys( configData );

        for ( var i = 0, l = configData[key].sections.length; i < l; i++ ) {
            var sectObjs = configData[key].sections[i];
            var ObjFiles = sectObjs.files;
            // console.log(sectObjs);

            if ( ObjFiles.indexOf( '*' ) > -1 ) {

                // files = glob.sync( ObjFiles, { nodir: true, matchBase:true } );
                files = glob.sync( ObjFiles );
                configData[key].sections[i].files = files;
                // console.log('I have a star', ObjFiles);
            }
            else {
                // var filesStat = fs.statSync( ObjFiles );
                // var filesTest = filesStat.isFile();
                // if ( filesTest === false ) {
                //     files = glob.sync( ObjFiles+'*', { nodir: true, matchBase:true } );
                //     configData[key].sections[i].files = files;
                // }

                //test if files ends in a slash 
                // console.log('obj', ObjFiles);
                var suffix = '/';
                var suffixLength = suffix.length;
                var filesLength = ObjFiles.length;
                // console.log('obj length', filesLength);
                var slashEnd = ObjFiles.indexOf( suffix, ( filesLength - suffixLength ) );
                // console.log('where the last slash is', slashEnd);
                // console.log( ObjFiles.indexOf( suffix, objFiles.length - suffix.length ) !== -1 );
                if( slashEnd === filesLength - 1) {
                    // console.log('we have and end match!', ObjFiles);
                    files = glob.sync( ObjFiles+'**/*' );
                    configData[key].sections[i].files = files;
                }
                else {
                    files = glob.sync( ObjFiles+'/**/*' );
                    configData[key].sections[i].files = files;
                    // console.log('i do not have an end slash', ObjFiles);
                }
            }
        }

        console.log(configData.documentation.sections);
    },
    readFile: function() {
        //get the data and throw it into a variable that we can use to edit
        this.configObj = configFile;

        // console.log(this.configObj);
        
        this.parseFiles();
    },
    parseFiles: function() {
        
        var sections = this.config;
        
        async.each( sections, this.parseFile, function() {
            
            console.log( 'DONE!', util.inspect( sections, { depth: 5, colors:true } ));

        });
    },
    parseFile: function( section, callback) {
        
        //internal function that parses using comment-parse on currentFile
        function parseComment( currentFile, data ) {
            
            // grab each comment block
            var comments = data.split( '/**' );
            
            // the first array item is always an empty string, so we shift
            comments.shift();
            
            
            for ( var i = 0; i < comments.length; i++ ) {
                
                // split blocks into comment and code content
                var block = comments[ i ].split(/^\s*\*\//m)
                    , toParse = '/**' + block[ 0 ] + ' */'
                    ;
                
                // add comment section to array
                comments[ i ] = commentParser( toParse )[ 0 ];
                
                // add code to data obj
                comments[ i ].code = block[ 1 ];
            }
            
            // now include path
            var push = {
                path: currentFile,
                data: comments
            };
            
            return push;
        }

        // only one file declared
        if ( typeof section.files === 'string' ) {
            
            var currentFile = section.files;
            
            fs.readFile( currentFile, { encoding: 'UTF8' }, function( err, data ) {
                
                section.files = [];
                section.files.push( parseComment( currentFile, data ));
                
                return callback( null );
            });
        }
        else {
            
            // array of files declared
            var files = section.files;
            section.files = [];
            
            async.each( files,
                function( item, callback ) {
                    
                    var currentFile = item;
                    
                    // read all files
                    fs.readFile( currentFile, { encoding: 'UTF8'}, function( err, data ) {
                        
                        section.files.push( parseComment( currentFile, data ));
                        
                        return callback( null );
                    });
                },
                function( err ) {

                    return callback( null );
                }
            );
        }
    }
};

module.exports.init();<commit_msg>testing patterns to work with updated changed from docs<commit_after>var configLocation = './notes/example-patterns-config';

var fs = require( 'fs' )
    , util = require( 'util' )
    , configFile = require( configLocation )
    , css = require( 'css' )
    , example = require( './notes/example-config')
    , async = require( 'async' )
    , commentParser = require( 'comment-parser' )
    ;
// run in the terminal using `node index.js`
module.exports = {

    configObj: {},

    config: require( './notes/parseFiles/input' ).patterns.sections,
    
    init: function( options ) {
        
        this.readFile();
        
        this.getFiles(this.configObj);
        
    },
    getFiles: function( data ) {

        var configData = data
            , files;
       
        var glob = require("glob");

        var key = Object.keys( configData );

        for ( var i = 0, l = configData[key].sections.length; i < l; i++ ) {
            var sectObjs = configData[key].sections[i];
            var ObjFiles = sectObjs.files;
            // console.log(sectObjs);

            if ( ObjFiles.indexOf( '*' ) > -1 ) {

                // files = glob.sync( ObjFiles, { nodir: true, matchBase:true } );
                files = glob.sync( ObjFiles );
                configData[key].sections[i].files = files;
                // console.log('I have a star', ObjFiles);
            }
            else {
                // var filesStat = fs.statSync( ObjFiles );
                // var filesTest = filesStat.isFile();
                // if ( filesTest === false ) {
                //     files = glob.sync( ObjFiles+'*', { nodir: true, matchBase:true } );
                //     configData[key].sections[i].files = files;
                // }

                //test if files ends in a slash 
                // console.log('obj', ObjFiles);
                var suffix = '/';
                var suffixLength = suffix.length;
                var filesLength = ObjFiles.length;
                // console.log('obj length', filesLength);
                var slashEnd = ObjFiles.indexOf( suffix, ( filesLength - suffixLength ) );
                // console.log('where the last slash is', slashEnd);
                // console.log( ObjFiles.indexOf( suffix, objFiles.length - suffix.length ) !== -1 );
                if( slashEnd === filesLength - 1) {
                    // console.log('we have and end match!', ObjFiles);
                    files = glob.sync( ObjFiles+'**/*' );
                    configData[key].sections[i].files = files;
                }
                else {
                    files = glob.sync( ObjFiles+'/**/*' );
                    configData[key].sections[i].files = files;
                    // console.log('i do not have an end slash', ObjFiles);
                }
            }
        }

        console.log(configData.patterns.sections);
    },
    readFile: function() {
        //get the data and throw it into a variable that we can use to edit
        this.configObj = configFile;

        // console.log(this.configObj);
        
        // this.parseFiles();
    },
    parseFiles: function() {
        
        var sections = this.config;
        
        async.each( sections, this.parseFile, function() {
            
            console.log( 'DONE!', util.inspect( sections, { depth: 5, colors:true } ));

        });
    },
    parseFile: function( section, callback) {
        
        //internal function that parses using comment-parse on currentFile
        function parseComment( currentFile, data ) {
            
            // grab each comment block
            var comments = data.split( '/**' );
            
            // the first array item is always an empty string, so we shift
            comments.shift();
            
            
            for ( var i = 0; i < comments.length; i++ ) {
                
                // split blocks into comment and code content
                var block = comments[ i ].split(/^\s*\*\//m)
                    , toParse = '/**' + block[ 0 ] + ' */'
                    ;
                
                // add comment section to array
                comments[ i ] = commentParser( toParse )[ 0 ];
                
                // add code to data obj
                comments[ i ].code = block[ 1 ];
            }
            
            // now include path
            var push = {
                path: currentFile,
                data: comments
            };
            
            return push;
        }

        // only one file declared
        if ( typeof section.files === 'string' ) {
            
            var currentFile = section.files;
            
            fs.readFile( currentFile, { encoding: 'UTF8' }, function( err, data ) {
                
                section.files = [];
                section.files.push( parseComment( currentFile, data ));
                
                return callback( null );
            });
        }
        else {
            
            // array of files declared
            var files = section.files;
            section.files = [];
            
            async.each( files,
                function( item, callback ) {
                    
                    var currentFile = item;
                    
                    // read all files
                    fs.readFile( currentFile, { encoding: 'UTF8'}, function( err, data ) {
                        
                        section.files.push( parseComment( currentFile, data ));
                        
                        return callback( null );
                    });
                },
                function( err ) {

                    return callback( null );
                }
            );
        }
    }
};

module.exports.init();
{'fraction_non_alphanumeric': 0.1161424127203595, 'fraction_numerical': 0.0019011406844106464, 'mean_word_length': 1.1721096096096095, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1473366', 'n_tokens_mistral': 3003, 'n_tokens_neox': 2834, 'n_words': 749}