<commit_before>var hp = require('./headerparser.js');
var TextParser = function() {
};

/**
* static method to extract parameters form text segment
* @{param} 
* */
TextParser.parseText = function(fd, txtStart, txtEnd) {

	var buffer = new Buffer(txtEnd - txtStart),
		str = fs.readSync(fd, buffer, 0, buffer.length, null).toString('utf8');
		params = {},
		re = /[\\\|\/\x00-\x1F]\$(\w+)[\\\|\/\x00-\x1F]([a-zA-Z0-9 \.\,\-:_]+)/g,
		arr
	;
	
  while ((arr = myRe.exec(str)) !== null) {
    params[arr[1].toLowerCase()] = isNaN(+arr[2]) ? arr[2] : +arr[2];
  }
  return params;
};

module.exports = TextParser;<commit_msg>text parser<commit_after>var TextParser = function() {
};

/**
* static method to extract parameters form text segment
* @param {file descriptor}
* @param {Object} header object - will be be ammended if any further data is
* 	discovered in text segment
*
* @returns {object} parameter set for fcs file
* */
TextParser.parseText = function(fd, header) {

	var buffer = new Buffer(header.text.end - header.text.start),
		str = fs.readSync(fd, buffer, 0, buffer.length, null).toString('utf8');
		params = {},
		rE = /[\\\|\/\x00-\x1F]\$(\w+)[\\\|\/\x00-\x1F]([a-zA-Z0-9 \.\,\-:_]+)/g,
		arr
	;
	
  while ((arr = rE.exec(str)) !== null) {
    params[arr[1].toLowerCase()] = isNaN(+arr[2]) ? arr[2] : +arr[2];
  }
	if(+params.begindata && +params.enddata) {
		header.data.start = +params.begindata;
		header.data.end = +params.enddata;
	}
	if(+params.beginanalysis && +params.endanalysis) {
		header.analysis.start = +params.beginanalysis;
		header.analysis.end = +params.endanalysis;
	}
  return params;
};

module.exports = TextParser;
{'fraction_non_alphanumeric': 0.20858524788391777, 'fraction_numerical': 0.016928657799274487, 'mean_word_length': 3.5972222222222223, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6465694', 'n_tokens_mistral': 641, 'n_tokens_neox': 609, 'n_words': 128}