<commit_before>'use strict';

var utils = require('../utils');

module.exports = function( _source ) {

    var source = _source.replace(/\r\n/g, '\n');

    var parent = null;
    var tokens = null;
    var blocks = null;

    console.log( source );

    return {
        parent: parent,
        tokens: tokens,
        blocks: blocks
    };
};<commit_msg>updates to the parse function<commit_after>'use strict';

var utils = require('../utils');

var returnArrayOfClasses = function( _string ){
    // splits into: ['class="', 'classes', '"']
    var origional = _string.split('"');
    return origional[1].trim().split(/\s+/);
};

module.exports = function( _source ) {

    var source = _source.replace(/\r\n/g, '\n');

    var i = 0;
    var parent = null;
    var tokens = [];
    var blocks = {};
    var inRaw = false;
    var line = 1;
    var stack = [];
    var anyChar = '[\\s\\S]*?';
    // Split the template source based on class attrs
    // /(class="[\\s\\S]*?")/
    var classSplitter = new RegExp(
      '(class="' + anyChar + '")'
    );
    var splitSource = source.split(classSplitter);

    /*!
    * Loop over the source, split via the tag/var/comment regular expression splitter.
    * Send each chunk to the appropriate parser.
    */

    var e = 0;

    utils.each( splitSource, function ( _chunk ) {

        var isClass = _chunk.includes('class="');

        if (!_chunk || !isClass) {
            e++;
            return;
        }

        // only initialise vars if needed
        // save the origional string for later
        var origionalString = _chunk;
        var classes = returnArrayOfClasses( _chunk );

        var matches = [];
        var unnededClasses = '';

        for ( i = 0; i < classes.length; i++) {
            if( classes[ i ].includes('#') ){
                matches.push( classes[ i ] );
            }else{
                unnededClasses = unnededClasses + classes[ i ] + ' ';
            }
        }

        console.log('-------');
        console.log( matches );
        console.log( unnededClasses );
        console.log( origionalString );
        console.log('-------');
        tokens.push({
            index: e,
            newClass: 'class="' + unnededClasses + '"'
        });
        e++;
    });

    for ( i = 0; i < tokens.length; i++) {
        console.log(tokens[i].index);
        splitSource[ tokens[i].index ] = tokens[i].newClass;
    }
    console.log('----------');
    console.log(splitSource.join(' '));

    return {
        parent: parent,
        tokens: tokens,
        blocks: blocks
    };
};
{'fraction_non_alphanumeric': 0.16388996512979465, 'fraction_numerical': 0.002324680356450988, 'mean_word_length': 1.8498896247240617, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3584499', 'n_tokens_mistral': 791, 'n_tokens_neox': 761, 'n_words': 205}