<commit_before>--[[--
Module for utf8 string operations.

This is a LuaJIT FFI wrapper for utf8proc.

@module ffi.utf8proc
]]

local ffi = require("ffi")
local C = ffi.C

require("ffi/posix_h")
require("ffi/utf8proc_h")

local libutf8proc
if ffi.os == "Windows" then
    libutf8proc = ffi.load("libs/libutf8proc-2.dll")
elseif ffi.os == "OSX" then
    libutf8proc = ffi.load("libs/libutf8proc.2.dylib")
else
    libutf8proc = ffi.load("libs/libutf8proc.so.2")
end

local Utf8Proc = {}

--- Lowercases an utf8-encoded string
--- @string str string to lowercase
--- @bool normalize normalizes the string during operation
--- @treturn string the lowercased string
function Utf8Proc.lowercase(str, normalize)
    if normalize == nil then normalize = true end

    if normalize then
        return Utf8Proc.lowercase_NFKC_Casefold(str)
    else
        return Utf8Proc.lowercase_dumb(str)
    end
end

-- with normalization
function Utf8Proc.lowercase_NFKC_Casefold(str)
    local folded_strz = libutf8proc.utf8proc_NFKC_Casefold(str)
    local folded_str = ffi.string(folded_strz)
    C.free(folded_strz)
    return folded_str
end

-- no normalization here
function Utf8Proc.lowercase_dumb(str)
    local lowercased = ""
    local tmp_str = (" "):rep(10)
    local tmp_p = ffi.cast("utf8proc_uint8_t *", tmp_str)
    local str_p = ffi.cast("const utf8proc_uint8_t *", str)
    local codepoint = ffi.new("utf8proc_int32_t[1]")
    local count = 0
    local pos = 0
    local str_len = #str -- may contain NUL
    while pos < str_len do
        -- get codepoint
        local bytes = libutf8proc.utf8proc_iterate(str_p + pos, -1, codepoint)
        -- lowercase codepoint
        local lower_cp = libutf8proc.utf8proc_tolower(codepoint[0])
        -- encode lowercased codepoint and get length of new char*
        local lower_len = libutf8proc.utf8proc_encode_char(lower_cp, tmp_p)
        tmp_p[lower_len] = 0
        -- append
        lowercased = lowercased .. ffi.string(tmp_p)

        if bytes > 0 then
            count = count + 1
            pos = pos + bytes
        else
            return lowercased
        end
    end
    return lowercased
end

--- Normalizes an utf8-encoded string
--- @string str string to lowercase
--- @treturn string the normalized string
function Utf8Proc.normalize_NFC(str)
    local normalized_strz = libutf8proc.utf8proc_NFC(str)
    local normalized_str = ffi.string(normalized_strz)
    C.free(normalized_strz)
    return normalized_str<commit_msg>ffi/utf8proc: add uppercase conversion helper (#1528)<commit_after>--[[--
Module for utf8 string operations.

This is a LuaJIT FFI wrapper for utf8proc.

@module ffi.utf8proc
]]

local ffi = require("ffi")
local C = ffi.C

require("ffi/posix_h")
require("ffi/utf8proc_h")

local libutf8proc
if ffi.os == "Windows" then
    libutf8proc = ffi.load("libs/libutf8proc-2.dll")
elseif ffi.os == "OSX" then
    libutf8proc = ffi.load("libs/libutf8proc.2.dylib")
else
    libutf8proc = ffi.load("libs/libutf8proc.so.2")
end

local Utf8Proc = {}

--- Lowercases an utf8-encoded string
--- @string str string to lowercase
--- @bool normalize normalizes the string during operation
--- @treturn string the lowercased string
function Utf8Proc.lowercase(str, normalize)
    if normalize == nil then normalize = true end

    if normalize then
        return Utf8Proc.lowercase_NFKC_Casefold(str)
    else
        return Utf8Proc.cased_dumb(str, true)
    end
end

-- with normalization
function Utf8Proc.lowercase_NFKC_Casefold(str)
    local folded_strz = libutf8proc.utf8proc_NFKC_Casefold(str)
    local folded_str = ffi.string(folded_strz)
    C.free(folded_strz)
    return folded_str
end

-- no normalization here
function Utf8Proc.lowercase_dumb(str)
    return Utf8Proc.cased_dumb(str, true)
end

function Utf8Proc.uppercase_dumb(str)
    return Utf8Proc.cased_dumb(str, false)
end

function Utf8Proc.cased_dumb(str, is_lower)
    local cased = ""
    local tmp_str = (" "):rep(10)
    local tmp_p = ffi.cast("utf8proc_uint8_t *", tmp_str)
    local str_p = ffi.cast("const utf8proc_uint8_t *", str)
    local codepoint = ffi.new("utf8proc_int32_t[1]")
    local count = 0
    local pos = 0
    local str_len = #str -- may contain NUL
    while pos < str_len do
        -- get codepoint
        local bytes = libutf8proc.utf8proc_iterate(str_p + pos, -1, codepoint)
        -- cased codepoint
        local cp = is_lower and libutf8proc.utf8proc_tolower(codepoint[0]) or libutf8proc.utf8proc_toupper(codepoint[0])
        -- encode cased codepoint and get length of new char*
        local len = libutf8proc.utf8proc_encode_char(cp, tmp_p)
        tmp_p[len] = 0
        -- append
        cased = cased .. ffi.string(tmp_p)

        if bytes > 0 then
            count = count + 1
            pos = pos + bytes
        else
            return cased
        end
    end
    return cased
end

--- Normalizes an utf8-encoded string
--- @string str string to lowercase
--- @treturn string the normalized string
function Utf8Proc.normalize_NFC(str)
    local normalized_strz = libutf8proc.utf8proc_NFC(str)
    local normalized_str = ffi.string(normalized_strz)
    C.free(normalized_strz)
    return normalized_str
{'fraction_non_alphanumeric': 0.09441233140655106, 'fraction_numerical': 0.02157996146435453, 'mean_word_length': 3.1003159557661926, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6564289', 'n_tokens_mistral': 1850, 'n_tokens_neox': 1812, 'n_words': 446}