<commit_before>__precompile__()
module TinySegmenter

export Segmenter, tokenize

typealias US UTF8String

macro t_str(s)
  tuple(s...)
end

# Use out of range of Unicode code point. See also: https://en.wikipedia.org/wiki/Code_point
const B1 = Char(0x110001)
const B2 = Char(0x110002)
const B3 = Char(0x110003)
const E1 = Char(0x110004)
const E2 = Char(0x110005)
const E3 = Char(0x110006)

const BIAS = -332

const BC1 = Dict{Tuple{UInt8,UInt8},Int}(t"HH" => 6, t"II" => 2, t"OH" => -1378)
const BC2 = Dict{Tuple{UInt8,UInt8},Int}(t"AA" => -3267, t"AI" => 2744, t"AN" => -878, t"HH" => -4070, t"HM" => -1711, t"HN" => 4012, t"HO" => 3761, t"IA" => 1327, t"IH" => -1184, t"II" => -1332, t"IK" => 1721, t"IO" => 5492, t"KI" => 3831, t"KK" => -8741, t"MH" => -3132, t"MK" => 3334, t"OO" => -2920)
const BC3 = Dict{Tuple{UInt8,UInt8},Int}(t"HH" => 996, t"HI" => 626, t"HK" => -721, t"HN" => -1307, t"HO" => -836, t"IH" => -301, t"KK" => 2762, t"MK" => 1079, t"MM" => 4034, t"OA" => -1652, t"OH" => 266)
const BP1 = Dict{Tuple{UInt8,UInt8},Int}(t"BB" => 295, t"OB" => 304, t"OO" => -125, t"UB" => 352)
const BP2 = Dict{Tuple{UInt8,UInt8},Int}(t"BO" => 60, t"OO" => -1762)<commit_msg>Segmenter doesn't exist any more, so don't export it<commit_after>__precompile__()
module TinySegmenter

export tokenize

typealias US UTF8String

macro t_str(s)
  tuple(s...)
end

# Use out of range of Unicode code point. See also: https://en.wikipedia.org/wiki/Code_point
const B1 = Char(0x110001)
const B2 = Char(0x110002)
const B3 = Char(0x110003)
const E1 = Char(0x110004)
const E2 = Char(0x110005)
const E3 = Char(0x110006)

const BIAS = -332

const BC1 = Dict{Tuple{UInt8,UInt8},Int}(t"HH" => 6, t"II" => 2, t"OH" => -1378)
const BC2 = Dict{Tuple{UInt8,UInt8},Int}(t"AA" => -3267, t"AI" => 2744, t"AN" => -878, t"HH" => -4070, t"HM" => -1711, t"HN" => 4012, t"HO" => 3761, t"IA" => 1327, t"IH" => -1184, t"II" => -1332, t"IK" => 1721, t"IO" => 5492, t"KI" => 3831, t"KK" => -8741, t"MH" => -3132, t"MK" => 3334, t"OO" => -2920)
const BC3 = Dict{Tuple{UInt8,UInt8},Int}(t"HH" => 996, t"HI" => 626, t"HK" => -721, t"HN" => -1307, t"HO" => -836, t"IH" => -301, t"KK" => 2762, t"MK" => 1079, t"MM" => 4034, t"OA" => -1652, t"OH" => 266)
const BP1 = Dict{Tuple{UInt8,UInt8},Int}(t"BB" => 295, t"OB" => 304, t"OO" => -125, t"UB" => 352)
const BP2 = Dict{Tuple{UInt8,UInt8},Int}(t"BO" => 60, t"OO" => -1762)
{'fraction_non_alphanumeric': 0.23973176865046103, 'fraction_numerical': 0.16429170159262363, 'mean_word_length': 4.474770642201835, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 77, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1162586', 'n_tokens_mistral': 1383, 'n_tokens_neox': 1115, 'n_words': 264}