<commit_before>require "helpers"
local pp = require "purepack"
require "conf"

local function name(metric_,step_,period_)
      return string.format("%s;%s:%s",metric_,
                           secs_to_time_unit(step_),
                           secs_to_time_unit(period_))
end

local nop = function() end
local return_0 = function() return 0 end

local NOP_SEQUENCE = {
  slots = function() return {} end,
    name = function() return "" end,
    metric = return_0,
    step = return_0,
    period = return_0,
    slot = nop,
    slot_index = return_0,
    update = nop,
    update_batch = nop,
    latest = nop,
    latest_timestamp = return_0,
    reset = nop,
    serialize = nop,
         }

function sequence(db_,name_)
  local _metric,_step,_period,_name,_seq_storage

  local function at(idx_,offset_,a_,b_,c_)
    -- a nil offset_ is translated to reading/writing the entire cell (3 items)
    if not a_ then
      return _seq_storage.get_slot(idx_,offset_)
    end
    _seq_storage.set_slot(idx_,offset_,a_,b_,c_)
  end

  local function get_timestamp(idx_)
    return at(idx_,0)
  end

  local function get_hits(idx_)
    return at(idx_,1)
  end

  local function get_sum(idx_)
    return at(idx_,2)
  end


  local function set_slot(idx_,timestamp_,hits_,sum_)
    at(idx_,nil,timestamp_,hits_,sum_)
  end

  local function latest(idx_)
    local pos = (_period/_step)
    if not idx_ then
      return at(pos,0)
    end
    at(pos,0,idx_)
  end

  local function latest_timestamp()
    return get_timestamp(latest())
  end

  local function reset()
    _seq_storage.reset()
    _seq_storage.save()
  end

  _name = name_

  _metric,_step,_period = split_name(name_)
  if not (_metric and _step and _period ) then
    loge("failed generating sequence",name_)
    return NOP_SEQUENCE
  end

  _seq_storage = db_.sequence_storage(name_,_period/_step)


  local function update(timestamp_,sum_,hits_,replace_)
    local idx,adjusted_timestamp = calculate_idx(timestamp_,_step,_period)
    -- if this value is way back (but still fits in this slot)
    -- we discard it
    local timestamp,hits,sum = at(idx)

    if adjusted_timestamp<timestamp then
      return
    end
    -- we need to check whether we should update the current slot
    -- or if are way ahead of the previous time the slot was updated
    -- over-write its value

    if not replace_ and adjusted_timestamp==timestamp and hits>0 then
      -- no need to worry about the latest here, as we have the same (adjusted) timestamp
      hits,sum = hits+(hits_ or 1), sum+sum_
    else
      hits,sum = hits_ or 1,sum_
    end
    set_slot(idx,adjusted_timestamp,hits,sum)

    if adjusted_timestamp>latest_timestamp() then
      latest(idx)
    end

    _seq_storage.save(_name)
    return adjusted_timestamp,sum
  end

  local function update_batch(slots_)
    -- slots is a flat array of arrays
    local j = 1
    local match = string.match
    while j<#slots_ do
      local sum,hits,timestamp = slots_[j],tonumber(slots_[j+1]),tonumber(slots_[j+2])
      j = j + 3
      local replace,s = match(sum,"(=?)(%d+)")
      update(timestamp,s,hits,replace)
    end

    _seq_storage.save(_name)
    return j-1
  end


  local function indices(sorted_)
    -- the sorted list is of indices to the main one
    local array = {}
    for i=1,(_period/_step) do
      array[i] = i-1
    end
    if sorted_ then
      table.sort(array,
                 function(u,v)
                   return get_timestamp(u)<get_timestamp(v)
                 end)
    end

    return array
  end

  local function serialize(opts_,metric_cb_,slot_cb_)

    local function serialize_slot(idx_,skip_empty_,slot_cb_)
      local timestamp,hits,sum = at(idx_)
      if not skip_empty_ or sum~=0 or hits~=0 or timestamp~=0 then
        slot_cb_(sum,hits,timestamp)
      end
    end

    opts_ = opts_ or {}
    metric_cb_(_metric,_step,_period)

    local now,latest_ts = time_now(),latest_timestamp()
    local min_timestamp = (opts_.filter=="latest" and latest_ts-_period) or
      (opts_.filter=="now" and now-_period) or nil


    if opts_.deep then
      for j,s in ipairs(indices(opts_.sorted)) do
        if not min_timestamp or min_timestamp<get_timestamp(s) then
          serialize_slot(s,opts_.skip_empty,slot_cb_)
        end
      end
      return
    end

    if opts_.timestamps then
      for _,t in ipairs(opts_.timestamps) do
        if t=="*" then
          for _,s in ipairs(indices(opts_.sorted)) do
            serialize_slot(s,nil,slot_cb_)
          end
        else
          local ts = to_timestamp(t,now,latest_ts)
          if ts then
            if type(ts)=="number" then
              ts = {ts,ts+_step-1}
            end
            for t = ts[1],ts[2],(ts[1]<ts[2] and _step or -_step) do
              local idx,_ = calculate_idx(t,_step,_period)
              local ts = get_timestamp(idx)
              if t-ts<_period and (not min_timestamp or min_timestamp<ts) then
                serialize_slot(idx,nil,slot_cb_)
              end
            end
          end
        end
      end
    end
  end

  local function slot(idx_)<commit_msg>some sort optimizations<commit_after>require "helpers"
local pp = require "purepack"
require "conf"

local function name(metric_,step_,period_)
      return string.format("%s;%s:%s",metric_,
                           secs_to_time_unit(step_),
                           secs_to_time_unit(period_))
end

local nop = function() end
local return_0 = function() return 0 end

local NOP_SEQUENCE = {
  slots = function() return {} end,
    name = function() return "" end,
    metric = return_0,
    step = return_0,
    period = return_0,
    slot = nop,
    slot_index = return_0,
    update = nop,
    update_batch = nop,
    latest = nop,
    latest_timestamp = return_0,
    reset = nop,
    serialize = nop,
         }

function sequence(db_,name_)
  local _metric,_step,_period,_name,_seq_storage

  local function at(idx_,offset_,a_,b_,c_)
    -- a nil offset_ is translated to reading/writing the entire cell (3 items)
    if not a_ then
      return _seq_storage.get_slot(idx_,offset_)
    end
    _seq_storage.set_slot(idx_,offset_,a_,b_,c_)
  end

  local function get_timestamp(idx_)
    return at(idx_,0)
  end

  local function get_hits(idx_)
    return at(idx_,1)
  end

  local function get_sum(idx_)
    return at(idx_,2)
  end


  local function set_slot(idx_,timestamp_,hits_,sum_)
    at(idx_,nil,timestamp_,hits_,sum_)
  end

  local function latest(idx_)
    local pos = (_period/_step)
    if not idx_ then
      return at(pos,0)
    end
    at(pos,0,idx_)
  end

  local function latest_timestamp()
    return get_timestamp(latest())
  end

  local function reset()
    _seq_storage.reset()
    _seq_storage.save()
  end

  _name = name_

  _metric,_step,_period = split_name(name_)
  if not (_metric and _step and _period ) then
    loge("failed generating sequence",name_)
    return NOP_SEQUENCE
  end

  _seq_storage = db_.sequence_storage(name_,_period/_step)


  local function update(timestamp_,sum_,hits_,replace_)
    local idx,adjusted_timestamp = calculate_idx(timestamp_,_step,_period)
    -- if this value is way back (but still fits in this slot)
    -- we discard it
    local timestamp,hits,sum = at(idx)

    if adjusted_timestamp<timestamp then
      return
    end
    -- we need to check whether we should update the current slot
    -- or if are way ahead of the previous time the slot was updated
    -- over-write its value

    if not replace_ and adjusted_timestamp==timestamp and hits>0 then
      -- no need to worry about the latest here, as we have the same (adjusted) timestamp
      hits,sum = hits+(hits_ or 1), sum+sum_
    else
      hits,sum = hits_ or 1,sum_
    end
    set_slot(idx,adjusted_timestamp,hits,sum)

    if adjusted_timestamp>latest_timestamp() then
      latest(idx)
    end

    _seq_storage.save(_name)
    return adjusted_timestamp,sum
  end

  local function update_batch(slots_)
    -- slots is a flat array of arrays
    local j = 1
    local match = string.match
    while j<#slots_ do
      local sum,hits,timestamp = slots_[j],tonumber(slots_[j+1]),tonumber(slots_[j+2])
      j = j + 3
      local replace,s = match(sum,"(=?)(%d+)")
      update(timestamp,s,hits,replace)
    end

    _seq_storage.save(_name)
    return j-1
  end


  local function indices(sorted_)
    if not sorted_ then
      return coroutine.wrap(
        function()
          for i=1,(_period/_step) do
            coroutine.yield(i-1)
          end
        end)
    end
    local array = {}
    for i=1,(_period/_step) do
      array[i] = {i-1,get_timestamp(i-1)}
    end
    table.sort(array, function(u,v) return u[2]<v[2] end)
    return coroutine.wrap(
      function()
        for _,s in ipairs(array) do
          coroutine.yield(s[1])
        end
      end)
  end

  local function serialize(opts_,metric_cb_,slot_cb_)

    local function serialize_slot(idx_,skip_empty_,slot_cb_)
      local timestamp,hits,sum = at(idx_)
      if not skip_empty_ or sum~=0 or hits~=0 or timestamp~=0 then
        slot_cb_(sum,hits,timestamp)
      end
    end

    opts_ = opts_ or {}
    metric_cb_(_metric,_step,_period)

    local now,latest_ts = time_now(),latest_timestamp()
    local min_timestamp = (opts_.filter=="latest" and latest_ts-_period) or
      (opts_.filter=="now" and now-_period) or nil


    if opts_.deep then
      for s in indices(opts_.sorted) do
        if not min_timestamp or min_timestamp<get_timestamp(s) then
          serialize_slot(s,opts_.skip_empty,slot_cb_)
        end
      end
      return
    end

    if opts_.timestamps then
      for _,t in ipairs(opts_.timestamps) do
        if t=="*" then
          for s in indices(opts_.sorted) do
            serialize_slot(s,nil,slot_cb_)
          end
        else
          local ts = to_timestamp(t,now,latest_ts)
          if ts then
            if type(ts)=="number" then
              ts = {ts,ts+_step-1}
            end
            for t = ts[1],ts[2],(ts[1]<ts[2] and _step or -_step) do
              local idx,_ = calculate_idx(t,_step,_period)
              local its = get_timestamp(idx)
              if t-its<_period and (not min_timestamp or min_timestamp<its) then
                serialize_slot(idx,nil,slot_cb_)
              end
            end
          end
        end
      end
    end
  end

  local function slot(idx_)
{'fraction_non_alphanumeric': 0.09048076923076923, 'fraction_numerical': 0.006538461538461538, 'mean_word_length': 2.5571135430916554, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '558803', 'n_tokens_mistral': 3387, 'n_tokens_neox': 3293, 'n_words': 902}