<commit_before>#ifndef BUFFERS_H
#define BUFFERS_H

#define CONDOR_IO_BUF_SIZE 4096

void sanity_check();

class Buf {
public:
	Buf(int sz=CONDOR_IO_BUF_SIZE);
	~Buf();

	inline void reset() { _dta_pt = _dta_sz = 0; }
	inline void rewind() { _dta_pt = 0; }


	inline int max_size() const { return _dta_maxsz; }
	inline int num_untouched() const { return _dta_sz - _dta_pt; }
	inline int num_touched() const { return _dta_pt; }
	inline int num_used() const { return _dta_sz; }
	inline int num_free() const { return _dta_maxsz - _dta_sz; }

	inline int empty() const { return _dta_sz == 0; }
	inline int full() const { return _dta_sz == _dta_maxsz; }
	inline int consumed() const { return _dta_pt == _dta_sz; }


	int write_to_fd(int fd, int sz=-1);
	int read_frm_fd(int fd, int sz=-1);

	int flush_to_fd(int fd, void * hdr=0, int sz=0);

	int put_max(const void *, int);
	int get_max(void *, int);

	int find(char);
	int peek(char &);
	int seek(int);

	inline Buf *next() const { return _next; }
	inline void set_next(Buf *b) { _next = b; }
	inline void *get_ptr() const { return &_dta[num_touched()]; }

private:

	char	*_dta;
	int		_dta_sz;
	int		_dta_maxsz;
	int		_dta_pt;
	Buf		*_next;
};




class ChainBuf {
public:
	ChainBuf() : _head(0), _tail(0), _curr(0),_tmp(0) {}
	~ChainBuf() { if (_tmp) delete [] _tmp; }


	inline int consumed() { return !_tail || (_tail && _tail->consumed()); }

	int put(Buf *);
	void reset();

	int get(void *, int);
	int get_tmp(void *&, char);

	int peek(char &);

private:
	Buf			*_head;
	Buf			*_tail;
	Buf			*_curr;
	char		*_tmp;
};




#endif
<commit_msg>fixed memory leak - chainbuf destructor now calls reset to free many buffers<commit_after>#ifndef BUFFERS_H
#define BUFFERS_H

#define CONDOR_IO_BUF_SIZE 4096

void sanity_check();

class Buf {
public:
	Buf(int sz=CONDOR_IO_BUF_SIZE);
	~Buf();

	inline void reset() { _dta_pt = _dta_sz = 0; }
	inline void rewind() { _dta_pt = 0; }


	inline int max_size() const { return _dta_maxsz; }
	inline int num_untouched() const { return _dta_sz - _dta_pt; }
	inline int num_touched() const { return _dta_pt; }
	inline int num_used() const { return _dta_sz; }
	inline int num_free() const { return _dta_maxsz - _dta_sz; }

	inline int empty() const { return _dta_sz == 0; }
	inline int full() const { return _dta_sz == _dta_maxsz; }
	inline int consumed() const { return _dta_pt == _dta_sz; }


	int write_to_fd(int fd, int sz=-1);
	int read_frm_fd(int fd, int sz=-1);

	int flush_to_fd(int fd, void * hdr=0, int sz=0);

	int put_max(const void *, int);
	int get_max(void *, int);

	int find(char);
	int peek(char &);
	int seek(int);

	inline Buf *next() const { return _next; }
	inline void set_next(Buf *b) { _next = b; }
	inline void *get_ptr() const { return &_dta[num_touched()]; }

private:

	char	*_dta;
	int		_dta_sz;
	int		_dta_maxsz;
	int		_dta_pt;
	Buf		*_next;
};




class ChainBuf {
public:
	ChainBuf() : _head(0), _tail(0), _curr(0),_tmp(0) {}
	~ChainBuf() { reset(); } 	// reset() in destructor to delete all bufs

	inline int consumed() { return !_tail || (_tail && _tail->consumed()); }

	int put(Buf *);
	void reset();

	int get(void *, int);
	int get_tmp(void *&, char);

	int peek(char &);

private:
	Buf			*_head;
	Buf			*_tail;
	Buf			*_curr;
	char		*_tmp;
};




#endif

{'fraction_non_alphanumeric': 0.1423780487804878, 'fraction_numerical': 0.009146341463414634, 'mean_word_length': 3.971212121212121, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5033434', 'n_tokens_mistral': 1430, 'n_tokens_neox': 1352, 'n_words': 373}