<commit_before>/*
    CSV parser
    1st argument: input data file name
**/
/*
 sep {String} character used as separator
 toNumber {Boolean} try to convert values to a Number
 quotedData {Boolean} data items may be double-quoted
 linesToDisplay {Integer} number of lines to show
**/
var sep = ';'
var toNumber = true
var quotedData = true
var linesToDisplay = 10

toNumber = false
quotedData = false
sep = '\t'
linesToDisplay = 0

var fs = require('fs')
var fileName = process.argv[2]
var file = fs.createReadStream(fileName)

var Tokenizer = require('..')
var tok = new Tokenizer       // Main parser
var stringtok = new Tokenizer // String processing - replaces "" with "

// Variables used by the parsers handlers
var data, firstLine, currentString
<commit_msg>Fixed leftover in examples/csv.js<commit_after>/*
    CSV parser
    1st argument: input data file name
**/
/*
 sep {String} character used as separator
 toNumber {Boolean} try to convert values to a Number
 quotedData {Boolean} data items may be double-quoted
 linesToDisplay {Integer} number of lines to show
**/
var sep = ';'
var toNumber = true
var quotedData = true
var linesToDisplay = 10

//-----------------------------------------------------------------------------
var fs = require('fs')
var fileName = process.argv[2]
var file = fs.createReadStream(fileName)

var Tokenizer = require('..')
var tok = new Tokenizer       // Main parser
var stringtok = new Tokenizer // String processing - replaces "" with "

// Variables used by the parsers handlers
var data, firstLine, currentString

{'fraction_non_alphanumeric': 0.13837283792440744, 'fraction_numerical': 0.005765534913516977, 'mean_word_length': 3.0677083333333335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4774752', 'n_tokens_mistral': 437, 'n_tokens_neox': 427, 'n_words': 181}