<commit_before>// SPDX-License-Identifier: LGPL-2.1-or-later
/*
 * Copyright (C) 2022 <NAME> <<EMAIL>>
 */

#ifndef UTILS_GP_UTF_POS_H
#define UTILS_GP_UTF_POS_H

#include <utils/gp_utf.h>
#include <stdint.h>
#include <stddef.h>
#include <unistd.h>

/*
 * Position in an UTF-8 string.
 */
typedef struct gp_utf8_pos {
	size_t bytes;
	size_t chars;
} gp_utf8_pos;

static inline gp_utf8_pos gp_utf8_pos_first(void)
{
	return (gp_utf8_pos){0, 0};
}

static inline gp_utf8_pos gp_utf8_pos_last(const char *str)
{
	gp_utf8_pos ret = {0, 0};

	for (;;) {
		int8_t chsz = gp_utf8_next_chsz(str, ret.bytes);

		if (chsz <= 0)
			return ret;

		ret.bytes += chsz;
		ret.chars++;
	}
}

static inline int gp_utf8_pos_at_end(const char *str, gp_utf8_pos pos)
{
	return !str[pos.bytes];
}

static inline int gp_utf8_pos_at_home(gp_utf8_pos pos)
{
	return !pos.bytes;
}

static inline int gp_utf8_pos_eq(gp_utf8_pos a, gp_utf8_pos b)
{
	return a.bytes == b.bytes;
}

static inline int gp_utf8_pos_gt(gp_utf8_pos a, gp_utf8_pos b)
{
	return a.bytes > b.bytes;
}

static inline int gp_utf8_pos_ge(gp_utf8_pos a, gp_utf8_pos b)
{
	return a.bytes >= b.bytes;
}

static inline gp_utf8_pos gp_utf8_pos_sub(gp_utf8_pos a, gp_utf8_pos b)
{
	gp_utf8_pos res = {
		.bytes = a.bytes - b.bytes,
		.chars = a.chars - b.chars,
	};

	return res;
}

static inline gp_utf8_pos gp_utf8_pos_add(gp_utf8_pos a, gp_utf8_pos b)
{
	gp_utf8_pos res = {
		.bytes = a.bytes + b.bytes,
		.chars = a.chars + b.chars,
	};

	return res;
}

static inline gp_utf8_pos gp_utf8_pos_min(gp_utf8_pos a, gp_utf8_pos b)
{
	if (a.bytes < b.bytes)
		return a;
	else
		return b;
}

static inline gp_utf8_pos gp_utf8_pos_max(gp_utf8_pos a, gp_utf8_pos b)
{
	if (a.bytes > b.bytes)
		return a;
	else
		return b;
}

static inline ssize_t gp_utf8_pos_move(const char *str, gp_utf8_pos *cur_pos, ssize_t dir)
{
	ssize_t dirs = dir;

	if (dir > 0) {
		while (dir > 0) {
			int8_t chsz = gp_utf8_next_chsz(str, cur_pos->bytes);

			if (chsz <= 0)
				return dirs - dir;

			cur_pos->bytes += chsz;
			cur_pos->chars++;

			dir--;
		}

		return dirs;
	}

	if (dir < 0) {
		while (dir < 0) {
			int8_t chsz = gp_utf8_prev_chsz(str, cur_pos->bytes);

			if (chsz <= 0)
				return dir - dirs;

			cur_pos->bytes -= chsz;
			cur_pos->chars--;

			dir++;
		}

		return -dirs;
	}

	return 0;
}

#endif /* UTILS_GP_UTF_POS_H */
<commit_msg>utils: add two more utf8_pos helpers<commit_after>// SPDX-License-Identifier: LGPL-2.1-or-later
/*
 * Copyright (C) 2022 <NAME> <<EMAIL>>
 */

#ifndef UTILS_GP_UTF_POS_H
#define UTILS_GP_UTF_POS_H

#include <utils/gp_utf.h>
#include <stdint.h>
#include <stddef.h>
#include <unistd.h>

/*
 * Position in an UTF-8 string.
 */
typedef struct gp_utf8_pos {
	size_t bytes;
	size_t chars;
} gp_utf8_pos;

static inline gp_utf8_pos gp_utf8_pos_first(void)
{
	return (gp_utf8_pos){0, 0};
}

static inline gp_utf8_pos gp_utf8_pos_last(const char *str)
{
	gp_utf8_pos ret = {0, 0};

	for (;;) {
		int8_t chsz = gp_utf8_next_chsz(str, ret.bytes);

		if (chsz <= 0)
			return ret;

		ret.bytes += chsz;
		ret.chars++;
	}
}

static inline int gp_utf8_pos_at_end(const char *str, gp_utf8_pos pos)
{
	return !str[pos.bytes];
}

static inline int gp_utf8_pos_at_home(gp_utf8_pos pos)
{
	return !pos.bytes;
}

static inline int gp_utf8_pos_eq(gp_utf8_pos a, gp_utf8_pos b)
{
	return a.bytes == b.bytes;
}

static inline int gp_utf8_pos_gt(gp_utf8_pos a, gp_utf8_pos b)
{
	return a.bytes > b.bytes;
}

static inline int gp_utf8_pos_ge(gp_utf8_pos a, gp_utf8_pos b)
{
	return a.bytes >= b.bytes;
}

static inline gp_utf8_pos gp_utf8_pos_sub(gp_utf8_pos a, gp_utf8_pos b)
{
	gp_utf8_pos res = {
		.bytes = a.bytes - b.bytes,
		.chars = a.chars - b.chars,
	};

	return res;
}

static inline gp_utf8_pos gp_utf8_pos_add(gp_utf8_pos a, gp_utf8_pos b)
{
	gp_utf8_pos res = {
		.bytes = a.bytes + b.bytes,
		.chars = a.chars + b.chars,
	};

	return res;
}

static inline gp_utf8_pos gp_utf8_pos_min(gp_utf8_pos a, gp_utf8_pos b)
{
	if (a.bytes < b.bytes)
		return a;
	else
		return b;
}

static inline gp_utf8_pos gp_utf8_pos_max(gp_utf8_pos a, gp_utf8_pos b)
{
	if (a.bytes > b.bytes)
		return a;
	else
		return b;
}

static inline ssize_t gp_utf8_pos_move(const char *str, gp_utf8_pos *cur_pos, ssize_t dir)
{
	ssize_t dirs = dir;

	if (dir > 0) {
		while (dir > 0) {
			int8_t chsz = gp_utf8_next_chsz(str, cur_pos->bytes);

			if (chsz <= 0)
				return dirs - dir;

			cur_pos->bytes += chsz;
			cur_pos->chars++;

			dir--;
		}

		return dirs;
	}

	if (dir < 0) {
		while (dir < 0) {
			int8_t chsz = gp_utf8_prev_chsz(str, cur_pos->bytes);

			if (chsz <= 0)
				return dir - dirs;

			cur_pos->bytes -= chsz;
			cur_pos->chars--;

			dir++;
		}

		return -dirs;
	}

	return 0;
}

static inline uint32_t gp_utf8_pos_prev(const char *str, gp_utf8_pos *pos)
{
	int8_t chsz;

	if (!pos->bytes)
		return 0;

	chsz = gp_utf8_prev_chsz(str, pos->bytes);
	if (chsz <= 0)
		return 0;

	pos->bytes -= chsz;
	pos->chars--;

	str += pos->bytes;

	return gp_utf8_next(&str);
}

static inline uint32_t gp_utf8_pos_next(const char *str, gp_utf8_pos *pos)
{
	int8_t chsz;

	chsz = gp_utf8_next_chsz(str, pos->bytes);
	if (chsz <= 0)
		return 0;

	str += pos->bytes;

	pos->bytes += chsz;
	pos->chars++;

	return gp_utf8_next(&str);
}

#endif /* UTILS_GP_UTF_POS_H */

{'fraction_non_alphanumeric': 0.13178733031674209, 'fraction_numerical': 0.02865761689291101, 'mean_word_length': 3.589100346020761, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6480898', 'n_tokens_mistral': 2693, 'n_tokens_neox': 2435, 'n_words': 529}