<commit_before>local sha1 = require "luchador.sha1"
local serializer = require "luchador.serializer"
local zlib = require "zlib"
local namespace = 'LC_'

local Storage = {}
local mt = {__index = Storage}

function Storage.new(datastore, options)
  local storage = {datastore = datastore,
                   windowed_hit_count = 0,
                   page_key_filter = options.page_key_filter,
                   local_entity_size = (options.local_entity_size or 65536),
                   min_gzip_size = (options.min_gzip_size or 20),
                   min_hits_for_local = (options.min_hits_for_local or 50)}
  setmetatable(storage, mt)
  return storage
end

function Storage:page_key()
  local key = ngx.var.scheme .. "://" .. ngx.var.host .. ngx.var.request_uri

  if self.page_key_filter then
    key = self.page_key_filter(key, self.datastore)
  end

  return sha1.digest_base64(key)
end

function Storage:get_metadata(req_h)
  local metadata, locally_cached = self:get(self:page_key(), true)

  if metadata == nil then
    return nil
  else
    for _, metadata in pairs(metadata) do
      local cached_vary_key = metadata[1]
      local cached_req_h = metadata[2]
      local cached_resp_h = metadata[3]

      if cached_resp_h['Vary'] then
        if cached_vary_key == self.vary_key(cached_resp_h['Vary'], req_h) then
          return cached_resp_h, locally_cached
        end
      else
        return cached_resp_h, locally_cached
      end
    end
  end
end

function Storage:store_metadata(req_h, resp_h, digest_key, ttl)
  if resp_h["Date"] == nil then
    resp_h["Date"] = ngx.http_time(ngx.time())
  end

  resp_h['X-Content-Digest'] = digest_key
  resp_h['Set-Cookie'] = nil
  local k = self:page_key()
  local h = (self:get(k) or {})

  local vk = self.vary_key(resp_h['Vary'], req_h)
  local vary_position = 1
  for i,v in ipairs(h) do
    if v[1] == vk then
      vary_position = i
    else
      vary_position = i + 1
    end
  end

  local cached_vary_val = {vk, req_h, resp_h}
  h[vary_position] = cached_vary_val

  self:set(k, h, ttl, true)
end

function Storage.vary_key(vary, req_h)
  local vk = {}

  if vary then
    for h in vary:gmatch("[^ ,]+") do
      h = h:lower()
      table.insert(vk, req_h[h] or '')
    end
  end

  return table.concat(vk)
end

function Storage:get_page(metadata)
  if not (metadata == nil) then
    local digest = metadata["X-Content-Digest"]
    if not (digest == nil) then
      return self:get(digest, false)
    end
  end
end

function Storage:store_page(resp, req_h, ttl)
  ngx.log(ngx.INFO, "MISS" .. ngx.var.request_uri)

  local digest_key = ngx.md5(resp.body)

  self:store_metadata(req_h, resp.header, digest_key, ttl)
  self:set(digest_key, resp.body, ttl)
  return true
end

function Storage:compress(content, content_type, use_best)
  local compression_level = zlib.BEST_SPEED
  if use_best then
    compression_level = zlib.BEST_COMPRESSION
  end

  if content_type and
     (content_type:match('text') or content_type:match('application')) and
     #content > self.min_gzip_size
  then
    content = zlib.compress(content, compression_level, nil, 15+16)
    return content, "gzip"
  else
    return content, nil
  end
end

function Storage:get_skip()
  return ngx.shared.cache_metadata:get(self:page_key() .. 'skip')
end

function Storage:set_skip()
  local r, err = ngx.shared.cache_metadata:set(self:page_key() .. 'skip', true, 30)
  return r
end

function Storage:get_lock(timeout)
  local r, err = ngx.shared.cache_metadata:add(self:page_key() .. 'lock', true, timeout)
  return r
end

function Storage:release_lock()
  return ngx.shared.cache_metadata:delete(self:page_key() .. 'lock')
end

function Storage:set(key, val, ttl, is_metadata)
  key = namespace .. key

  val = {val = val, ttl = ttl, created = ngx.time()}
  val = serializer.serialize(val)

  self.datastore:set(key, val, ttl)
  if is_metadata then
    self:local_set(key, val, ttl, is_metadata)
  end
end

function Storage:incr_hit_count(key, ttl)
  total_key = key .. 'hits'
  window_key = key .. 'whits'
  local count = self:incr_or_set(total_key, ttl)

  -- hit window of 30 seconds for local cache
  local window_count = self:incr_or_set(window_key, 30)

  ngx.header['X-Hit-Count'] = count
  self.windowed_hit_count = window_count
end

function Storage:incr_or_set(key, ttl)
  local c = ngx.shared.cache_metadata:incr(key, 1)
<commit_msg>Average 60 sec hit count with 30 sec for windowed hit count<commit_after>local sha1 = require "luchador.sha1"
local serializer = require "luchador.serializer"
local zlib = require "zlib"
local namespace = 'LC_'

local Storage = {}
local mt = {__index = Storage}

function Storage.new(datastore, options)
  local storage = {datastore = datastore,
                   windowed_hit_count = 0,
                   page_key_filter = options.page_key_filter,
                   local_entity_size = (options.local_entity_size or 65536),
                   min_gzip_size = (options.min_gzip_size or 20),
                   min_hits_for_local = (options.min_hits_for_local or 50)}
  setmetatable(storage, mt)
  return storage
end

function Storage:page_key()
  local key = ngx.var.scheme .. "://" .. ngx.var.host .. ngx.var.request_uri

  if self.page_key_filter then
    key = self.page_key_filter(key, self.datastore)
  end

  return sha1.digest_base64(key)
end

function Storage:get_metadata(req_h)
  local metadata, locally_cached = self:get(self:page_key(), true)

  if metadata == nil then
    return nil
  else
    for _, metadata in pairs(metadata) do
      local cached_vary_key = metadata[1]
      local cached_req_h = metadata[2]
      local cached_resp_h = metadata[3]

      if cached_resp_h['Vary'] then
        if cached_vary_key == self.vary_key(cached_resp_h['Vary'], req_h) then
          return cached_resp_h, locally_cached
        end
      else
        return cached_resp_h, locally_cached
      end
    end
  end
end

function Storage:store_metadata(req_h, resp_h, digest_key, ttl)
  if resp_h["Date"] == nil then
    resp_h["Date"] = ngx.http_time(ngx.time())
  end

  resp_h['X-Content-Digest'] = digest_key
  resp_h['Set-Cookie'] = nil
  local k = self:page_key()
  local h = (self:get(k) or {})

  local vk = self.vary_key(resp_h['Vary'], req_h)
  local vary_position = 1
  for i,v in ipairs(h) do
    if v[1] == vk then
      vary_position = i
    else
      vary_position = i + 1
    end
  end

  local cached_vary_val = {vk, req_h, resp_h}
  h[vary_position] = cached_vary_val

  self:set(k, h, ttl, true)
end

function Storage.vary_key(vary, req_h)
  local vk = {}

  if vary then
    for h in vary:gmatch("[^ ,]+") do
      h = h:lower()
      table.insert(vk, req_h[h] or '')
    end
  end

  return table.concat(vk)
end

function Storage:get_page(metadata)
  if not (metadata == nil) then
    local digest = metadata["X-Content-Digest"]
    if not (digest == nil) then
      return self:get(digest, false)
    end
  end
end

function Storage:store_page(resp, req_h, ttl)
  ngx.log(ngx.INFO, "MISS" .. ngx.var.request_uri)

  local digest_key = ngx.md5(resp.body)

  self:store_metadata(req_h, resp.header, digest_key, ttl)
  self:set(digest_key, resp.body, ttl)
  return true
end

function Storage:compress(content, content_type, use_best)
  local compression_level = zlib.BEST_SPEED
  if use_best then
    compression_level = zlib.BEST_COMPRESSION
  end

  if content_type and
     (content_type:match('text') or content_type:match('application')) and
     #content > self.min_gzip_size
  then
    content = zlib.compress(content, compression_level, nil, 15+16)
    return content, "gzip"
  else
    return content, nil
  end
end

function Storage:get_skip()
  return ngx.shared.cache_metadata:get(self:page_key() .. 'skip')
end

function Storage:set_skip()
  local r, err = ngx.shared.cache_metadata:set(self:page_key() .. 'skip', true, 30)
  return r
end

function Storage:get_lock(timeout)
  local r, err = ngx.shared.cache_metadata:add(self:page_key() .. 'lock', true, timeout)
  return r
end

function Storage:release_lock()
  return ngx.shared.cache_metadata:delete(self:page_key() .. 'lock')
end

function Storage:set(key, val, ttl, is_metadata)
  key = namespace .. key

  val = {val = val, ttl = ttl, created = ngx.time()}
  val = serializer.serialize(val)

  self.datastore:set(key, val, ttl)
  if is_metadata then
    self:local_set(key, val, ttl, is_metadata)
  end
end

function Storage:incr_hit_count(key, ttl)
  local total_key = key .. 'hits'
  local window_key = key .. 'whits'
  local count = self:incr_or_set(total_key, ttl)

  -- hit window of 60 seconds for local cache
  local long_window_count = self:incr_or_set(window_key .. 'l', 60)
  local window_count = self:incr_or_set(window_key, 30)

  ngx.header['X-Hit-Count'] = count
  self.windowed_hit_count = (window_count + long_window_count) / 2
end

function Storage:incr_or_set(key, ttl)
  local c = ngx.shared.cache_metadata:incr(key, 1)

{'fraction_non_alphanumeric': 0.11015239802779023, 'fraction_numerical': 0.00818018825638727, 'mean_word_length': 3.512133468149646, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1746279', 'n_tokens_mistral': 3292, 'n_tokens_neox': 3167, 'n_words': 772}