<commit_before>/*
 * File:  strings.h
 * Copyright (C) 2004 The Institute for System Programming of the Russian Academy of Sciences (ISP RAS)
 */

#ifndef _STRINGS_H
#define _STRINGS_H

#include "sedna.h"

#include "tuple.h"
#include "e_string.h"
#include "micro.h"
#include "pstr_long.h"
///FIXME!!!!!!!!!!!!!!!!
// move text_type here, text_doc => text_pstr


//FIXME - length is pstr_long should not be int
#define STRING_ITERATOR_CALL_TEMPLATE_1tcptr(func, tcell_ptr, params) \
	switch (tcell_ptr->get_type()) \
	{ \
	case tc_light_atomic: {\
		char *str = tcell_ptr->get_str_mem(); \
		int len = tcell_ptr->get_strlen_mem(); \
		char_iterator start1(str, len, 0); \
		char_iterator end1(str, len, len); \
		func<char_iterator> params; \
		break; }\
	case tc_heavy_atomic_estr: \
	case tc_heavy_atomic_pstr_short: {\
		int len = tcell_ptr->get_strlen_vmm(); \
		xptr data = tcell_ptr->get_str_vmm(); \
\
		e_string_iterator_first start1(len, data); \
		e_string_iterator_first end1(0, data); \
		func<e_string_iterator> params; \
		break; }\
	case tc_heavy_atomic_pstr_long: {\
		int len = tcell_ptr->get_strlen_vmm(); \
		xptr data = tcell_ptr->get_str_vmm(); \
\
		pstr_long_iterator start1(data); \
		pstr_long_iterator end1(data, len); \
		func<pstr_long_iterator> params; \
		break; }\
	}

#define STRING_ITERATOR_CALL_TEMPLATE_1tcptr_3p(func, tcell_ptr, p1, p2, p3) STRING_ITERATOR_CALL_TEMPLATE_1tcptr(func, tcell_ptr, (start1, end1, p1, p2, p3) )
#define STRING_ITERATOR_CALL_TEMPLATE_1tcptr_4p(func, tcell_ptr, p1, p2, p3, p4) STRING_ITERATOR_CALL_TEMPLATE_1tcptr(func, tcell_ptr, (start1, end1, p1, p2, p3, p4))

class t_str_buf
{
private:
	xptr m_ptr;
	e_str m_estr;
	str_counted_ptr m_str_ptr;
	int m_len;//FIXME (don't use int type)
	char *m_buf;
	int m_buf_size;
	text_type m_ttype;
	static const int f_text_in_buf = 1;
	static const int f_text_in_estr_buf = 2;
	unsigned char m_flags;
	void move_to_mem_buf();
	void move_to_estr();
public:<commit_msg>strings.h fixes<commit_after>/*
 * File:  strings.h
 * Copyright (C) 2004 The Institute for System Programming of the Russian Academy of Sciences (ISP RAS)
 */

#ifndef _STRINGS_H
#define _STRINGS_H

#include "sedna.h"

#include "tuple.h"
#include "e_string.h"
#include "micro.h"
#include "pstr_long.h"
#include "e_string_o_iterator.h"
#include "e_string_iterator.h"
#include "char_iterator.h"
///FIXME!!!!!!!!!!!!!!!!
// move text_type here, text_doc => text_pstr


//FIXME - length is pstr_long should not be int
#define STRING_ITERATOR_CALL_TEMPLATE_1tcptr(func, tcell_ptr, params) \
	switch (tcell_ptr->get_type()) \
	{ \
	case tc_light_atomic: {\
		char *str = tcell_ptr->get_str_mem(); \
		int len = tcell_ptr->get_strlen_mem(); \
		char_iterator start1(str, len, 0); \
		char_iterator end1(str, len, len); \
		func<char_iterator> params; \
		break; }\
	case tc_heavy_atomic_estr: \
	case tc_heavy_atomic_pstr_short: {\
		int len = tcell_ptr->get_strlen_vmm(); \
		xptr data = tcell_ptr->get_str_vmm(); \
\
		e_string_iterator_first start1(len, data); \
		e_string_iterator_first end1(0, data); \
		func<e_string_iterator> params; \
		break; }\
	case tc_heavy_atomic_pstr_long: {\
		int len = tcell_ptr->get_strlen_vmm(); \
		xptr data = tcell_ptr->get_str_vmm(); \
\
		pstr_long_iterator start1(data); \
		pstr_long_iterator end1(data, len); \
		func<pstr_long_iterator> params; \
		break; }\
	}

#define STRING_ITERATOR_CALL_TEMPLATE_1tcptr_1p(func, tcell_ptr, p1) STRING_ITERATOR_CALL_TEMPLATE_1tcptr(func, (tcell_ptr), (start1, end1, p1) )
#define STRING_ITERATOR_CALL_TEMPLATE_1tcptr_2p(func, tcell_ptr, p1, p2) STRING_ITERATOR_CALL_TEMPLATE_1tcptr(func, (tcell_ptr), (start1, end1, p1, p2) )
#define STRING_ITERATOR_CALL_TEMPLATE_1tcptr_3p(func, tcell_ptr, p1, p2, p3) STRING_ITERATOR_CALL_TEMPLATE_1tcptr(func, (tcell_ptr), (start1, end1, p1, p2, p3) )
#define STRING_ITERATOR_CALL_TEMPLATE_1tcptr_4p(func, tcell_ptr, p1, p2, p3, p4) STRING_ITERATOR_CALL_TEMPLATE_1tcptr(func, (tcell_ptr), (start1, end1, p1, p2, p3, p4))

class t_str_buf
{
private:
	xptr m_ptr;
	e_str m_estr;
	str_counted_ptr m_str_ptr;
	int m_len;//FIXME (don't use int type)
	char *m_buf;
	int m_buf_size;
	text_type m_ttype;
	static const int f_text_in_buf = 1;
	static const int f_text_in_estr_buf = 2;
	unsigned char m_flags;
	void move_to_mem_buf();
	void move_to_estr();
public:
{'fraction_non_alphanumeric': 0.1270463454000461, 'fraction_numerical': 0.021673968180770116, 'mean_word_length': 5.144475920679887, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '786370', 'n_tokens_mistral': 1893, 'n_tokens_neox': 1841, 'n_words': 374}