<commit_before>#ifndef SSTRING_H
#define SSTRING_H
#ifdef __cplusplus
extern "C" {
#endif

/*
 * sstring.h
 *
 * #SHORTDOC string handling
 *
 * #DOC Provided functions allow efficient operations on strings. Internal
 * #DOC string format is binary, supporting arbitrary data. Operations
 * #DOC on strings involving format interpretation, e.g. string length is
 * #DOC interpreted as UTF-8 when calling to the Unicode function ss_len_u(),
 * #DOC and as raw data when calling the functions not using Unicode
 * #DOC interpretation (ss_len()/ss_size()). Strings below 256 bytes take just
 * #DOC 5 bytes for internal structure, and 5 * sizeof(size_t) for bigger
 * #DOC strings. Unicode size is cached between operations, when possible, so
 * #DOC in those cases UTF-8 string length computation would be O(1).
 *
 * Copyright (c) 2015-2019 <NAME>. All rights reserved.
 * Released under the BSD 3-Clause License (see the doc/LICENSE)
 */

#include "saux/scommon.h"
#include "saux/sdata.h"
#include "saux/shash.h"
#include "svector.h"

/*
 * String base structure
 *
 * Observations:
 * - The small string container is implemented using struct SDataSmall,
 *   defined in sdata.h. The compatibility is possible because both
 *   struct SDataSmall and struct SDataFull share the header, where
 *   is defined if using the smaller container or the full one.
 * - Usage of struct SDataFlags type-specific elements:
 *	flag1: Unicode size is cached
 *	flag2: string has UTF-8 encoding errors (e.g. after some operation)
 *	flag3: string reference (built using ss_cref[a]() or ss_ref[a]())
 *	flag4: string reference with C terminator (built using ss_cref[a]())
 */

struct SString {
	struct SDataFull d;
	size_t unicode_size;
};

struct SStringRef {
	struct SString s;
	const char *cstr;
};

struct SStringRefRW {
	struct SString s;
	char *str;
};

#define SS_RANGE (sizeof(size_t) - sizeof(srt_string))
#define EMPTY_SS                                                               \
	{                                                                      \
		EMPTY_SDataFull, 0                                             \
	}

/*
 * Types
 */

/* Opaque structures (accessors are provided) */<commit_msg>ss_max() bug fix: proper constant definition<commit_after>#ifndef SSTRING_H
#define SSTRING_H
#ifdef __cplusplus
extern "C" {
#endif

/*
 * sstring.h
 *
 * #SHORTDOC string handling
 *
 * #DOC Provided functions allow efficient operations on strings. Internal
 * #DOC string format is binary, supporting arbitrary data. Operations
 * #DOC on strings involving format interpretation, e.g. string length is
 * #DOC interpreted as UTF-8 when calling to the Unicode function ss_len_u(),
 * #DOC and as raw data when calling the functions not using Unicode
 * #DOC interpretation (ss_len()/ss_size()). Strings below 256 bytes take just
 * #DOC 5 bytes for internal structure, and 5 * sizeof(size_t) for bigger
 * #DOC strings. Unicode size is cached between operations, when possible, so
 * #DOC in those cases UTF-8 string length computation would be O(1).
 *
 * Copyright (c) 2015-2019 <NAME>. All rights reserved.
 * Released under the BSD 3-Clause License (see the doc/LICENSE)
 */

#include "saux/scommon.h"
#include "saux/sdata.h"
#include "saux/shash.h"
#include "svector.h"

/*
 * String base structure
 *
 * Observations:
 * - The small string container is implemented using struct SDataSmall,
 *   defined in sdata.h. The compatibility is possible because both
 *   struct SDataSmall and struct SDataFull share the header, where
 *   is defined if using the smaller container or the full one.
 * - Usage of struct SDataFlags type-specific elements:
 *	flag1: Unicode size is cached
 *	flag2: string has UTF-8 encoding errors (e.g. after some operation)
 *	flag3: string reference (built using ss_cref[a]() or ss_ref[a]())
 *	flag4: string reference with C terminator (built using ss_cref[a]())
 */

struct SString {
	struct SDataFull d;
	size_t unicode_size;
};

struct SStringRef {
	struct SString s;
	const char *cstr;
};

struct SStringRefRW {
	struct SString s;
	char *str;
};

#define SS_RANGE \
	(((size_t)-1) - S_MAX(sizeof(struct SString),	\
			      sizeof(struct SStringRef)))
#define EMPTY_SS                                                               \
	{                                                                      \
		EMPTY_SDataFull, 0                                             \
	}

/*
 * Types
 */

/* Opaque structures (accessors are provided) */
{'fraction_non_alphanumeric': 0.08940914158305463, 'fraction_numerical': 0.010479375696767, 'mean_word_length': 2.8276450511945392, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4174435', 'n_tokens_mistral': 1336, 'n_tokens_neox': 1270, 'n_words': 500}