<commit_before>local declarative_config = require("kong.db.schema.others.declarative_config")


local off = {}


local _mt = {}
_mt.__index = _mt


local function empty_list_cb()
  return {}
end


local function nil_cb()
  return nil
end


local function page_for_key(self, key, size, offset)
  offset = offset and tonumber(offset) or 1

  local cache = kong.cache
  if not cache then
    return {}
  end

  local list, err = cache:get(key, nil, empty_list_cb)
  if not list then
    return nil, err
  end

  local ret = {}
  local n = 1
  local name = self.schema.name
  for i = 0, size - 1 do
    local id = list[i + offset]
    if not id then
      offset = nil
      break
    end

    local ck = name .. ":" .. id .. "::::"
    local entry = cache:get(ck, nil, nil_cb)
    ret[n] = entry
    n = n + 1
  end

  if offset then
    return ret, nil, tostring(offset + size)
  end

  return ret
end


local function select_by_key(self, key)
  if not kong.cache then
    return nil
  end

  return kong.cache:get(key, nil, nil_cb)
end


local function page(self, size, offset)
  local key = self.schema.name .. "|list"
  return page_for_key(self, key, size, offset)<commit_msg>feat(db) encode offsets in `off` strategy in base64<commit_after>local declarative_config = require("kong.db.schema.others.declarative_config")


local kong = kong
local fmt = string.format
local tostring = tostring
local tonumber = tonumber
local encode_base64 = ngx.encode_base64
local decode_base64 = ngx.decode_base64


local off = {}


local _mt = {}
_mt.__index = _mt


local function empty_list_cb()
  return {}
end


local function nil_cb()
  return nil
end


local function page_for_key(self, key, size, offset)
  if offset then
    local token = decode_base64(offset)
    if not token then
      return nil, self.errors:invalid_offset(offset, "bad base64 encoding")
    end

    token = tonumber(token)
    if not token then
      return nil, self.errors:invalid_offset(offset, "invalid offset")
    end

    offset = token

  else
    offset = 1
  end

  local cache = kong.cache
  if not cache then
    return {}
  end

  local list, err = cache:get(key, nil, empty_list_cb)
  if not list then
    return nil, err
  end

  local ret = {}
  local n = 1
  local name = self.schema.name
  for i = 0, size - 1 do
    local id = list[i + offset]
    if not id then
      offset = nil
      break
    end

    local ck = name .. ":" .. id .. "::::"
    local entry = cache:get(ck, nil, nil_cb)
    ret[n] = entry
    n = n + 1
  end

  if offset then
    return ret, nil, encode_base64(tostring(offset + size), true)
  end

  return ret
end


local function select_by_key(self, key)
  if not kong.cache then
    return nil
  end

  return kong.cache:get(key, nil, nil_cb)
end


local function page(self, size, offset)
  local key = self.schema.name .. "|list"
  return page_for_key(self, key, size, offset)
{'fraction_non_alphanumeric': 0.09588092765662859, 'fraction_numerical': 0.008999653859466944, 'mean_word_length': 2.8688085676037485, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5301355', 'n_tokens_mistral': 1029, 'n_tokens_neox': 974, 'n_words': 330}