<commit_before>if ffi == nil then
    if require == nil then
        error('You must restart tarantool')
    end
    ffi = require('ffi')
    ffi.cdef('char * strchr(const char *s, int c);');
    ffi.cdef('char * strrchr(const char *s, int c);');
end

local DISABLE_SEARCH_BY_DOMAIN = true

function box.auto_increment_uniq(spaceno, uniq, ...)
    local tuple = box.select(spaceno, 1, uniq)

    if tuple then
        local format = 'i'

        if #tuple[0] == 8 then
            format = 'l'
        end

        return box.replace(spaceno, box.unpack(format, tuple[0]), uniq, ...)
    end

    return box.auto_increment(spaceno, uniq, ...)
end

local function find_level_domain_impl(email_domain_part, n)
    local t = {}
    local i = 1
    local point = ffi.C.strchr(email_domain_part, 46)
    while point ~= nil do
        t[i] = point
        i = i + 1
        point = ffi.C.strchr(point + 1, 46)
    end
    if #t < n - 1 then
        return nil
    end
    if #t == n - 1 then
        return email_domain_part
    end
    return ffi.string(t[#t - n + 1] + 1)
end

local function find_third_level_domain(email_domain_part)
    return find_level_domain_impl(email_domain_part, 3)
end

local function find_second_level_domain(email_domain_part)
    return find_level_domain_impl(email_domain_part, 2)
end

function string.ends(str, end_of_str)
   return end_of_str=='' or string.sub(str, -string.len(end_of_str)) == end_of_str
end

local function need_third_level(domain)
    local third_level_domains = {box.select(2, 0)}
    for _, i in pairs(third_level_domains) do
        if string.ends(domain, "." .. i[0]) then
            return true
        end
    end
    return false
end

function selist2_add_sender(mask, name_ru, name_en, cat)
    local point = ffi.C.strrchr(mask, 64)
    if point == nil then
        error('No dog in mask')
    end
    local email_domain_part = ffi.string(point + 1)

    local domain_to_store = ""
    if need_third_level(email_domain_part) then
        domain_to_store = find_third_level_domain(email_domain_part)
    else
        domain_to_store = find_second_level_domain(email_domain_part)
    end
    if not domain_to_store then
        error('Cannot find needed domain level')
    end
    return box.auto_increment_uniq(0, mask, name_ru, name_en, box.unpack('i', cat), domain_to_store):transform(5,1)
end

function get_sender_list_by_offset(offset, limit)
    offset = box.unpack('i', offset)
    limit = box.unpack('i', limit)

    local ret = {}
    local orig = {box.select_limit(0, 0, offset, limit)}
    for _, tuple in pairs(orig) do
        table.insert(ret, tuple:transform(5, 1))
    end

    return unpack(ret)
end

function get_sender_list_by_ids(...)
    local ret = { }
    for i, v in ipairs({...}) do
        local tuples = {box.select(0, 0, box.unpack('i', v))}
        for _, tuple in pairs(tuples) do
            table.insert(ret, tuple:transform(5, 1))
        end
    end
    return unpack(ret)
end

function selist2_search_by_mask(mask)
    local ret = { }
    local orig = {box.select(0, 1, mask)}
    for _, tuple in pairs(orig) do
        table.insert(ret, tuple:transform(5, 1))
    end
    return unpack(ret)
end

function selist2_search_by_domain(...)
    if DISABLE_SEARCH_BY_DOMAIN then
        return unpack({})
    end

    return _selist2_search_by_domain(...)
end

function selist2_search_by_domain_unswitchable(...)
    return _selist2_search_by_domain(...)
end

function _selist2_search_by_domain(domain)
    local domain_to_find = find_third_level_domain(domain)

    local ret = { }
    local ret_size = 0
    local orig = {}

    if domain_to_find then
        orig = {box.select(0, 2, domain_to_find)}
        for _, tuple in pairs(orig) do
            ret_size = ret_size + 1
            ret[ret_size] = tuple
        end
        if ret_size > 0 then
            return unpack(ret)
        end
    end

    domain_to_find = find_second_level_domain(domain)
    if not domain_to_find  then
        error('error string sent as domain')
    end

    orig = {box.select(0, 2, domain_to_find)}
    for _, tuple in pairs(orig) do
        ret_size = ret_size + 1
        ret[ret_size] = tuple
    end
    return unpack(ret)
end

function selist2_get_exceptions()
    return box.select(1, 0)
end<commit_msg>Selist bigger optimization<commit_after>if ffi == nil then
    if require == nil then
        error('You must restart tarantool')
    end
    ffi = require('ffi')
    ffi.cdef('char * strchr(const char *s, int c);');
    ffi.cdef('char * strrchr(const char *s, int c);');
end

local DISABLE_SEARCH_BY_DOMAIN = true
local ENTRIES_PER_DOMAIN = 101

function box.auto_increment_uniq(spaceno, uniq, ...)
    local tuple = box.select(spaceno, 1, uniq)

    if tuple then
        local format = 'i'

        if #tuple[0] == 8 then
            format = 'l'
        end

        return box.replace(spaceno, box.unpack(format, tuple[0]), uniq, ...)
    end

    return box.auto_increment(spaceno, uniq, ...)
end

local function find_level_domain_impl(email_domain_part, n)
    local t = {}
    local i = 1
    local point = ffi.C.strchr(email_domain_part, 46)
    while point ~= nil do
        t[i] = point
        i = i + 1
        point = ffi.C.strchr(point + 1, 46)
    end
    if #t < n - 1 then
        return nil
    end
    if #t == n - 1 then
        return email_domain_part
    end
    return ffi.string(t[#t - n + 1] + 1)
end

local function find_third_level_domain(email_domain_part)
    return find_level_domain_impl(email_domain_part, 3)
end

local function find_second_level_domain(email_domain_part)
    return find_level_domain_impl(email_domain_part, 2)
end

function string.ends(str, end_of_str)
   return end_of_str=='' or string.sub(str, -string.len(end_of_str)) == end_of_str
end

local function need_third_level(domain)
    local third_level_domains = {box.select(2, 0)}
    for _, i in pairs(third_level_domains) do
        if string.ends(domain, "." .. i[0]) then
            return true
        end
    end
    return false
end

function selist2_add_sender(mask, name_ru, name_en, cat)
    local point = ffi.C.strrchr(mask, 64)
    if point == nil then
        error('No dog in mask')
    end
    local email_domain_part = ffi.string(point + 1)

    local domain_to_store = ""
    if need_third_level(email_domain_part) then
        domain_to_store = find_third_level_domain(email_domain_part)
    else
        domain_to_store = find_second_level_domain(email_domain_part)
    end
    if not domain_to_store then
        error('Cannot find needed domain level')
    end
    return box.auto_increment_uniq(0, mask, name_ru, name_en, box.unpack('i', cat), domain_to_store):transform(5,1)
end

function get_sender_list_by_offset(offset, limit)
    offset = box.unpack('i', offset)
    limit = box.unpack('i', limit)

    local ret = {}
    local orig = {box.select_limit(0, 0, offset, limit)}
    for _, tuple in pairs(orig) do
        table.insert(ret, tuple:transform(5, 1))
    end

    return unpack(ret)
end

function get_sender_list_by_ids(...)
    local ret = { }
    for i, v in ipairs({...}) do
        local tuples = {box.select(0, 0, box.unpack('i', v))}
        for _, tuple in pairs(tuples) do
            table.insert(ret, tuple:transform(5, 1))
        end
    end
    return unpack(ret)
end

function selist2_search_by_mask(mask)
    local ret = { }
    local orig = {box.select(0, 1, mask)}
    for _, tuple in pairs(orig) do
        table.insert(ret, tuple:transform(5, 1))
    end
    return unpack(ret)
end

function selist2_search_by_domain(...)
    if DISABLE_SEARCH_BY_DOMAIN then
        return unpack({})
    end

    return _selist2_search_by_domain(...)
end

function selist2_search_by_domain_unswitchable(...)
    return _selist2_search_by_domain(...)
end

function _selist2_search_by_domain(domain)
    local domain_to_find = find_third_level_domain(domain)

    local ret = { }
    local ret_size = 0

    if domain_to_find then
        ret = {box.select_limit(0, 2, 0, ENTRIES_PER_DOMAIN, domain_to_find)}
        if #ret == ENTRIES_PER_DOMAIN then
            error('Limit reached by domain: ' .. domain_to_find)
        end
        if #ret > 0 then
            return ret
        end
    end

    domain_to_find = find_second_level_domain(domain)
    if not domain_to_find  then
        error('error string sent as domain')
    end

    ret = {box.select_limit(0, 2, 0, ENTRIES_PER_DOMAIN, domain_to_find)}
    if #ret == ENTRIES_PER_DOMAIN then
        error('Limit reached by domain: ' .. domain_to_find)
    end
    return ret
end

function selist2_get_exceptions()
    return box.select(1, 0)
end
{'fraction_non_alphanumeric': 0.10113531047265988, 'fraction_numerical': 0.013322520852641335, 'mean_word_length': 2.7913921826965304, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '555854', 'n_tokens_mistral': 3069, 'n_tokens_neox': 2956, 'n_words': 762}