<commit_before>package pony.text;

import pony.math.MathTools;
import pony.SPair;

#if macro
import haxe.macro.Context;
import haxe.macro.Expr;
import haxe.Serializer;
#end

@:enum abstract AnsiForeground(UInt) to UInt {
	var Default = 39;
	var Black = 30;
	var Red = 31;
	var Green = 32;
	var Yellow = 33;
	var Blue = 34;
	var Magenta = 35;
	var Cyan = 36;
	var LightGray = 37;
	var DarkGray = 90;
	var LightRed = 91;
	var LightGreen = 92;
	var LightYellow = 93;
	var LightBlue = 94;
	var LightMagenta = 95;
	var LightCyan = 96;
	var White = 97;
}

/**
 * TextTools
 * @author AxGord <<EMAIL>>
 */
class TextTools {

	public static inline var MODULE: String = 'pony.text.TextTools';
	public static var letters: Map<String, String> = [
		'en' => 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'ru' => 'АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ', 'num' => '0123456789'
	];
	private static inline var FIRST_ANSI_ID: Int = 192;

	public static inline function firstSplit(str: String, delimiter: String, ?startIndex: Int): SPair<String> {
		var index: Int = startIndex != null ? str.indexOf(delimiter, startIndex) : str.indexOf(delimiter);
		return index == -1 ? new SPair<String>(str, '') : new SPair<String>(str.substr(0, index), str.substr(index + 1));
	}

	public static inline function lastSplit(str: String, delimiter: String, ?startIndex: Int): SPair<String> {
		var index: Int = startIndex != null ? str.lastIndexOf(delimiter, startIndex) : str.lastIndexOf(delimiter);
		return index == -1 ? new SPair<String>(str, '') : new SPair<String>(str.substr(0, index), str.substr(index + 1));
	}

	public static inline function allAfter(str: String, delimiter: String, ?startIndex: Int): Null<String> {
		var index: Int = str.indexOf(delimiter, startIndex);
		return index == -1 ? null : str.substr(index + 1);
	}

	public static inline function allAfterLast(str: String, delimiter: String, ?startIndex: Int): Null<String> {
		var index: Int = startIndex == null ? str.lastIndexOf(delimiter) : str.lastIndexOf(delimiter, startIndex);
		return index == -1 ? null : str.substr(index + 1);
	}

	public static inline function allBefore(str: String, delimiter: String, ?startIndex: Int): Null<String> {
		var index: Int = str.indexOf(delimiter, startIndex);
		return index == -1 ? str : str.substr(0, index);
	}

	public static inline function allBeforeLast(str: String, delimiter: String, ?startIndex: Int): Null<String> {
		var index: Int = startIndex == null ? str.lastIndexOf(delimiter) : str.lastIndexOf(delimiter, startIndex);
		return index == -1 ? str : str.substr(0, index);
	}

	public static function onlyLetters(str: String, lang: String = 'en'): String {
		var ls: String = letters[lang];
		var result: String = '';
		for (i in 0...str.length) {
			var char: String = str.charAt(i);
			if (ls.indexOf(char) != -1) result += char;
		}
		return result;
	}

	public static function onlyLettersWithLower(str: String, lang: String = 'en'): String {
		var ls: String = letters[lang];
		var result: String = '';
		for (i in 0...str.length) {
			var char: String = str.charAt(i);
			if (ls.indexOf(char.toUpperCase()) != -1) result += char;<commit_msg>allAfter and allAfterLast delimiter length<commit_after>package pony.text;

import pony.math.MathTools;
import pony.SPair;

#if macro
import haxe.macro.Context;
import haxe.macro.Expr;
import haxe.Serializer;
#end

@:enum abstract AnsiForeground(UInt) to UInt {
	var Default = 39;
	var Black = 30;
	var Red = 31;
	var Green = 32;
	var Yellow = 33;
	var Blue = 34;
	var Magenta = 35;
	var Cyan = 36;
	var LightGray = 37;
	var DarkGray = 90;
	var LightRed = 91;
	var LightGreen = 92;
	var LightYellow = 93;
	var LightBlue = 94;
	var LightMagenta = 95;
	var LightCyan = 96;
	var White = 97;
}

/**
 * TextTools
 * @author AxGord <<EMAIL>>
 */
class TextTools {

	public static inline var MODULE: String = 'pony.text.TextTools';
	public static var letters: Map<String, String> = [
		'en' => 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'ru' => 'АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ', 'num' => '0123456789'
	];
	private static inline var FIRST_ANSI_ID: Int = 192;

	public static inline function firstSplit(str: String, delimiter: String, ?startIndex: Int): SPair<String> {
		var index: Int = startIndex != null ? str.indexOf(delimiter, startIndex) : str.indexOf(delimiter);
		return index == -1 ? new SPair<String>(str, '') : new SPair<String>(str.substr(0, index), str.substr(index + 1));
	}

	public static inline function lastSplit(str: String, delimiter: String, ?startIndex: Int): SPair<String> {
		var index: Int = startIndex != null ? str.lastIndexOf(delimiter, startIndex) : str.lastIndexOf(delimiter);
		return index == -1 ? new SPair<String>(str, '') : new SPair<String>(str.substr(0, index), str.substr(index + 1));
	}

	public static inline function allAfter(str: String, delimiter: String, ?startIndex: Int): Null<String> {
		var index: Int = str.indexOf(delimiter, startIndex);
		return index == -1 ? null : str.substr(index + delimiter.length);
	}

	public static inline function allAfterLast(str: String, delimiter: String, ?startIndex: Int): Null<String> {
		var index: Int = startIndex == null ? str.lastIndexOf(delimiter) : str.lastIndexOf(delimiter, startIndex);
		return index == -1 ? null : str.substr(index + delimiter.length);
	}

	public static inline function allBefore(str: String, delimiter: String, ?startIndex: Int): Null<String> {
		var index: Int = str.indexOf(delimiter, startIndex);
		return index == -1 ? str : str.substr(0, index);
	}

	public static inline function allBeforeLast(str: String, delimiter: String, ?startIndex: Int): Null<String> {
		var index: Int = startIndex == null ? str.lastIndexOf(delimiter) : str.lastIndexOf(delimiter, startIndex);
		return index == -1 ? str : str.substr(0, index);
	}

	public static function onlyLetters(str: String, lang: String = 'en'): String {
		var ls: String = letters[lang];
		var result: String = '';
		for (i in 0...str.length) {
			var char: String = str.charAt(i);
			if (ls.indexOf(char) != -1) result += char;
		}
		return result;
	}

	public static function onlyLettersWithLower(str: String, lang: String = 'en'): String {
		var ls: String = letters[lang];
		var result: String = '';
		for (i in 0...str.length) {
			var char: String = str.charAt(i);
			if (ls.indexOf(char.toUpperCase()) != -1) result += char;
{'fraction_non_alphanumeric': 0.141519911153419, 'fraction_numerical': 0.020307789941297796, 'mean_word_length': 4.420464316423044, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3190472', 'n_tokens_mistral': 2349, 'n_tokens_neox': 2179, 'n_words': 633}