<commit_before>/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2013 <NAME> (gsomix)
 */

#include <cstdio>
#include <shogun/io/LineReader.h>

using namespace shogun;

CLineReader::CLineReader()
	: m_stream(NULL), m_max_line_length(0), m_next_line_length(-1)
{
	m_buffer=new CCircularBuffer(0);
}

CLineReader::CLineReader(FILE* stream, char delimiter)
	: m_stream(stream), m_max_line_length(10*1024*1024), m_next_line_length(-1)
{
	m_buffer=new CCircularBuffer(m_max_line_length);
	m_tokenizer=new CDelimiterTokenizer();
	m_tokenizer->delimiters[delimiter]=1;
	m_buffer->set_tokenizer(m_tokenizer);
}

CLineReader::CLineReader(int32_t max_line_length, FILE* stream, char delimiter)
	: m_stream(stream), m_max_line_length(max_line_length), m_next_line_length(-1)
{
	m_buffer=new CCircularBuffer(m_max_line_length);<commit_msg>fixed an uninitialised memory error<commit_after>/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2013 <NAME> (gsomix)
 */

#include <cstdio>
#include <shogun/io/LineReader.h>

using namespace shogun;

CLineReader::CLineReader()
	: m_stream(NULL), m_max_line_length(0), m_next_line_length(-1)
{
	m_buffer=new CCircularBuffer(0);
	m_tokenizer=NULL;
}

CLineReader::CLineReader(FILE* stream, char delimiter)
	: m_stream(stream), m_max_line_length(10*1024*1024), m_next_line_length(-1)
{
	m_buffer=new CCircularBuffer(m_max_line_length);
	m_tokenizer=new CDelimiterTokenizer();
	m_tokenizer->delimiters[delimiter]=1;
	m_buffer->set_tokenizer(m_tokenizer);
}

CLineReader::CLineReader(int32_t max_line_length, FILE* stream, char delimiter)
	: m_stream(stream), m_max_line_length(max_line_length), m_next_line_length(-1)
{
	m_buffer=new CCircularBuffer(m_max_line_length);
{'fraction_non_alphanumeric': 0.1102288650163475, 'fraction_numerical': 0.02148528724894909, 'mean_word_length': 6.962825278810409, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4526623', 'n_tokens_mistral': 800, 'n_tokens_neox': 771, 'n_words': 172}