<commit_before>-- Copyright (C) 2013 <NAME>
-- @author <NAME> <<EMAIL>>

-- small overlay over shdict, smart cache load mechanism

-- TODO (rdsharma): remove this line once we have tracked down bugs
jit.off(true, true)

local _M = {}

local resty_lock = require("resty.lock")

-- TODO (jbyers): unwind from https://github.com/cloudflare/lua-resty-shcache
local DEBUG = false

-- defaults in secs
local DEFAULT_POSITIVE_TTL = 10     -- cache for, successful lookup
local DEFAULT_NEGATIVE_TTL = 2      -- cache for, failed lookup
local DEFAULT_ACTUALIZE_TTL = 2     -- stale data, actualize data for

-- default lock options, in secs
local DEFAULT_LOCK_EXPTIME = 1      -- max wait if failing to call unlock()
local DEFAULT_LOCK_TIMEOUT = 0.5    -- max waiting time of lock()
local DEFAULT_LOCK_MAXSTEP = 0.1    -- max sleeping interval



local bit = require("bit")
local band = bit.band
local bor = bit.bor
local st_format = string.format

-- there are only really 5 states total
-- is_stale    is_neg  is_from_cache
local MISS_STATE = 0             -- 0           0       0
local HIT_POSITIVE_STATE = 1     -- 0           0       1
local HIT_NEGATIVE_STATE = 3     -- 0           1       1
local STALE_POSITIVE_STATE = 5   -- 1           0       1
<commit_msg>Remove debug variable<commit_after>-- Copyright (C) 2013 <NAME>
-- @author <NAME> <<EMAIL>>

-- small overlay over shdict, smart cache load mechanism

-- TODO (rdsharma): remove this line once we have tracked down bugs
jit.off(true, true)

local _M = {}

local resty_lock = require("resty.lock")

-- defaults in secs
local DEFAULT_POSITIVE_TTL = 10     -- cache for, successful lookup
local DEFAULT_NEGATIVE_TTL = 2      -- cache for, failed lookup
local DEFAULT_ACTUALIZE_TTL = 2     -- stale data, actualize data for

-- default lock options, in secs
local DEFAULT_LOCK_EXPTIME = 1      -- max wait if failing to call unlock()
local DEFAULT_LOCK_TIMEOUT = 0.5    -- max waiting time of lock()
local DEFAULT_LOCK_MAXSTEP = 0.1    -- max sleeping interval



local bit = require("bit")
local band = bit.band
local bor = bit.bor
local st_format = string.format

-- there are only really 5 states total
-- is_stale    is_neg  is_from_cache
local MISS_STATE = 0             -- 0           0       0
local HIT_POSITIVE_STATE = 1     -- 0           0       1
local HIT_NEGATIVE_STATE = 3     -- 0           1       1
local STALE_POSITIVE_STATE = 5   -- 1           0       1

{'fraction_non_alphanumeric': 0.08638420403126286, 'fraction_numerical': 0.024681201151789386, 'mean_word_length': 2.5093795093795093, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '462405', 'n_tokens_mistral': 805, 'n_tokens_neox': 749, 'n_words': 278}