<commit_before>/*!
 * csv-stream
 * Copyright(c) 2012 HipSnip Limited
 * Author <NAME> <<EMAIL>>
 * MIT Licensed
 */

/**
 * Modules dependencies
 */

var EventEmitter = require('events').EventEmitter,
	util = require('util');

module.exports = Parser;

function Parser(options){
	EventEmitter.call(this);
	this.delimiter = options ? options.delimiter || ',' : ',';
	this.endLine = options ? options.endLine || '\n' : '\n';
	this.enclosedChar = options ? options.enclosedChar || '' : '';
	this.escapeChar = options ? options.escapeChar || '' : '';

	this._defaultColumns =  options ? !!options.columns : false;
	this.columns = options ? options.columns || [] : [];
	this._currentColumn = 0;
	this._index = 0;
	this._line = {};
	this._text = '';
	this._enclosing = null;
}


// Inherits from EventEmitter
util.inherits(Parser,EventEmitter);

Parser.prototype.end = function(s){
	if(s) this.parse(s);
	if(this._text || Object.getOwnPropertyNames(this._line).length){
		if(this._text.charAt(this._text.length -1) === '\r') this._text = this._text.slice(0,this._text.length - 1);
		this.emit('column',this.columns[this._currentColumn],this._text);
		this._line[this.columns[this._currentColumn]] = this._text;
		this.emit('data',this._line);
	}
	this.emit('end');
}

Parser.prototype.parse = function(s){
	for(var i = 0; i < s.length; i++){
		var c = s[i];
		if(this.escapeChar === c && this._enclosing && s[i+1] === this.enclosedChar){
			i++;
			this._text = this._text + s[i];
		}else{
			if(this.enclosedChar === c){
				this._enclosing = !this._enclosing;
			}else if(this.delimiter === c){
				if(this._enclosing){
					this._text = this._text + c;
				}else{
					if(this._index === 0 && !this._defaultColumns){
						this.columns[this._currentColumn] = this._text;
					}else{
						this.emit('column',this.columns[this._currentColumn],this._text);
						this._line[this.columns[this._currentColumn]] = this._text;
					}
					this._text = '';
					this._currentColumn++;
				}
			}else if(this.endLine === c){ //LF
				if(this._enclosing){
					this._text = this._text + c;
				}else{
					if(this._text.charAt(this._text.length -1) === '\r') this._text = this._text.slice(0,this._text.length - 1);
					if(this._index === 0 && !this._defaultColumns){
						this.columns[this._currentColumn] = this._text;
					}else{
						this.emit('column',this.columns[this._currentColumn],this._text);
						this._line[this.columns[this._currentColumn]] = this._text;
						this.emit('data',this._line);
					}
					this._index++;
					this._currentColumn = 0;
					this._line = {};
					this._text = '';
				}
			}else{
				this._text = this._text + c;
			}
		}
		
	}
}

<commit_msg>FIX: Replace charAt by [].<commit_after>/*!
 * csv-stream
 * Copyright(c) 2012 HipSnip Limited
 * Author <NAME> <<EMAIL>>
 * MIT Licensed
 */

/**
 * Modules dependencies
 */

var EventEmitter = require('events').EventEmitter,
	util = require('util');

module.exports = Parser;

function Parser(options){
	EventEmitter.call(this);
	this.delimiter = options ? options.delimiter || ',' : ',';
	this.endLine = options ? options.endLine || '\n' : '\n';
	this.enclosedChar = options ? options.enclosedChar || '' : '';
	this.escapeChar = options ? options.escapeChar || '' : '';

	this._defaultColumns =  options ? !!options.columns : false;
	this.columns = options ? options.columns || [] : [];
	this._currentColumn = 0;
	this._index = 0;
	this._line = {};
	this._text = '';
	this._enclosing = null;
}


// Inherits from EventEmitter
util.inherits(Parser,EventEmitter);

Parser.prototype.end = function(s){
	if(s) this.parse(s);
	if(this._text || Object.getOwnPropertyNames(this._line).length){
		if(this._text[this._text.length -1] === '\r') this._text = this._text.slice(0,this._text.length - 1);
		this.emit('column',this.columns[this._currentColumn],this._text);
		this._line[this.columns[this._currentColumn]] = this._text;
		this.emit('data',this._line);
	}
	this.emit('end');
}

Parser.prototype.parse = function(s){
	for(var i = 0; i < s.length; i++){
		var c = s[i];
		if(this.escapeChar === c && this._enclosing && s[i+1] === this.enclosedChar){
			i++;
			this._text = this._text + s[i];
		}else{
			if(this.enclosedChar === c){
				this._enclosing = !this._enclosing;
			}else if(this.delimiter === c){
				if(this._enclosing){
					this._text = this._text + c;
				}else{
					if(this._index === 0 && !this._defaultColumns){
						this.columns[this._currentColumn] = this._text;
					}else{
						this.emit('column',this.columns[this._currentColumn],this._text);
						this._line[this.columns[this._currentColumn]] = this._text;
					}
					this._text = '';
					this._currentColumn++;
				}
			}else if(this.endLine === c){ //LF
				if(this._enclosing){
					this._text = this._text + c;
				}else{
					if(this._text[this._text.length -1] === '\r') this._text = this._text.slice(0,this._text.length - 1);
					if(this._index === 0 && !this._defaultColumns){
						this.columns[this._currentColumn] = this._text;
					}else{
						this.emit('column',this.columns[this._currentColumn],this._text);
						this._line[this.columns[this._currentColumn]] = this._text;
						this.emit('data',this._line);
					}
					this._index++;
					this._currentColumn = 0;
					this._line = {};
					this._text = '';
				}
			}else{
				this._text = this._text + c;
			}
		}
		
	}
}
{'fraction_non_alphanumeric': 0.18762848065782098, 'fraction_numerical': 0.006353952532236965, 'mean_word_length': 3.372549019607843, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3655341', 'n_tokens_mistral': 2159, 'n_tokens_neox': 1839, 'n_words': 306}