<commit_before>var fs = require('fs')
    , util = require('util')
    , stream = require('stream')
    , es = require('event-stream');

// var parse = require('./parse.js')
var usfmValidate = require('./usfmValidate.js')

var input = './Tests/3JN.usfm';

var output = [];
var lines = [];
var count = 0;

var s = fs.createReadStream(input)
    .pipe(es.split())
    .pipe(es.mapSync(function(line){
        count ++;
      
        //object element for each line
        var data = {} ;
        var dataArr = line.split("\\");
        dataArr = dataArr.splice(1, dataArr.length);

        // loop through each lines
        // for(var i=0; i<dataArr.length; i++){
        if(dataArr.length != 0){
           
            //marker and values from each line
            data.marker = dataArr[0].split(" ")[0] ; 
           
            // check if values null after each marker
            // this works for lines which only contains a single marker with no text following eg: \p
            if(dataArr[0].split(/^[^ ]+ /)[1] !== undefined){
                data.value = dataArr[0].split(/^[^ ]+ /)[1]
            }
            else{
                data.value = "null";
            }
            if (data.marker == 'v'){
                data.number = dataArr[0].split(" ")[1]; 
                data.value = dataArr[0].split(/\d+ /)[1];
            }
            
            // handle inline markers as children of the corresponding line
            data.children = [];

            if(dataArr.length > 1){
                childArr = dataArr.slice(1);

                //inline markers structure starts from here
                for (var j = 0; j < childArr.length; j++) {
                    let split = childArr[j].split(" ");
                    // object for inline markers
                    // ternary operator handles if only markers are exist with out any values
                    data.children.push({
                        marker: split[0],
                        value: ((split[1] == "" ||split[1] == undefined) ? null : split.slice(1).join(" "))
                    });
                }
            }
            else{
                data.children = "null";
            }
            // count holds the line numbers in each line
            data.count = count;
            lines.push(data);  
        }
        else{
            console.log("newline")
        }
    })
    .on('error', function(){
        console.log('Error while reading file.');
    })
    .on('end', function(){
        console.log(lines)
        // usfmValidate.Validate(lines)
        // console.log('Read entire file.')
    })
);
module.exports = s;
<commit_msg>working fine for all initial single tags<commit_after>var fs = require('fs')
    , util = require('util')
    , stream = require('stream')
    , es = require('event-stream');

// var parse = require('./parse.js')
var usfmValidate = require('./usfmValidate.js')

var input = './Tests/3JN.usfm';

var output = [];
var lines = [];
var count = 0;

var s = fs.createReadStream(input)
    .pipe(es.split())
    .pipe(es.mapSync(function(line){
        count ++;
      
        //object element for each line
        var data = {} ;
        var dataArr = line.split("\\");
        dataArr = dataArr.splice(1, dataArr.length);

        // if the line is not empty
        if(dataArr.length != 0){
           
            //marker and values from each line
            data.marker = dataArr[0].split(" ")[0] ; 
           
            // check if values null after each marker
            // this works for lines which only contains a single marker with no text following eg: \p
            if(dataArr[0].split(/^[^ ]+ /)[1] !== undefined){
                data.value = dataArr[0].split(/^[^ ]+ /)[1]
            }
            else{
                data.value = "null";
            }
            if (data.marker == 'v'){
                data.number = dataArr[0].split(" ")[1]; 
                data.value = dataArr[0].split(/\d+ /)[1];
            }
            
            // handle inline markers as children of the corresponding line
            // data.children = [];
            child = [];

            if(dataArr.length > 1){
                childArr = dataArr.slice(1);

                //inline markers structure starts from here
                for (var j = 0; j < childArr.length; j++) {
                    let split = childArr[j].split(" ");
                    // object for inline markers
                    // ternary operator handles if only markers exist with out any values
                    child.push({
                        marker: split[0],
                        value: ((split[1] == "" ||split[1] == undefined) ? "null" : split.slice(1).join(" "))
                    });
                }
            }
            else{
                data.children = "null";
            }
            // count holds the line numbers in each line
            data.count = count;
            lines.push(data);  
        }
    })
    .on('error', function(){
        console.log('Error while reading file.');
    })
    .on('end', function(){
        console.log(lines)
        // usfmValidate.Validate(lines)
        // console.log('Read entire file.')
    })
);
module.exports = s;

{'fraction_non_alphanumeric': 0.14302059496567507, 'fraction_numerical': 0.008199847444698703, 'mean_word_length': 1.40045766590389, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '598040', 'n_tokens_mistral': 1463, 'n_tokens_neox': 1408, 'n_words': 382}