<commit_before>#include "FTindex.h"
#include "tr_globals.h"
#include "dtsearch.h"
#include "uhdd.h"
//using namespace dtSearch;

void ftlog_file::write_xptr_sequence(xptr_sequence* seq)
{
	int count = seq->size();
	this->write_data(&count, sizeof(count));
	xptr_sequence::iterator it=seq->begin();
	while (it!=seq->end())
	{
		xptr ptr = *it;
		this->write_data(&ptr, sizeof(ptr));
	}
}
int ftlog_file::skip_xptr_sequence()
{
	int count, res;
	res = this->read_data(&count, sizeof(count));
	if (res == 0)
		return 0;
	res = this->seek(this->next_lsn + sizeof(xptr) * count);
	return res;
}

xptr_sequence *ftlog_file::read_xptr_sequence()
{
	xptr_sequence *seq;
	int count, res;
	res = this->read_data(&count, sizeof(count));
	if (res == 0)
		return NULL;
<commit_msg>full-text index log fix<commit_after>#include "FTindex.h"
#include "tr_globals.h"
#include "dtsearch.h"
#include "uhdd.h"
//using namespace dtSearch;

void ftlog_file::write_xptr_sequence(xptr_sequence* seq)
{
	int count = seq->size();
	this->write_data(&count, sizeof(count));
	xptr_sequence::iterator it=seq->begin();
	while (it!=seq->end())
	{
		xptr ptr = *it;
		it++;
		this->write_data(&ptr, sizeof(ptr));
	}
}
int ftlog_file::skip_xptr_sequence()
{
	int count, res;
	res = this->read_data(&count, sizeof(count));
	if (res == 0)
		return 0;
	res = this->seek(this->next_lsn + sizeof(xptr) * count);
	return res;
}

xptr_sequence *ftlog_file::read_xptr_sequence()
{
	xptr_sequence *seq;
	int count, res;
	res = this->read_data(&count, sizeof(count));
	if (res == 0)
		return NULL;

{'fraction_non_alphanumeric': 0.17631917631917632, 'fraction_numerical': 0.003861003861003861, 'mean_word_length': 5.372950819672131, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2609569', 'n_tokens_mistral': 631, 'n_tokens_neox': 615, 'n_words': 128}