<commit_before>local Date = require('./build/date')
local os = require('os')

local formats = {
  '%Y-%m-%d %H:%M:%S UTC%z',
  '%a, %b %d %y %H:%M:%S GMT%z',
  '%a, %b %d %y %H:%M:%S UTC%z',
  '%a, %b %d %y %H:%M:%S',
  '%a, %b %d %Y %H:%M:%S GMT%z',
  '%a, %b %d %Y %H:%M:%S UTC%z',
  '%a, %b %d %Y %H:%M:%S',
  '%Y-%m-%d %H:%M:%S GMT%z',
  '%Y-%m-%d %H:%M:%S',
  '%Y-%m-%dT%H:%M:%SZ%z',
  '%Y-%m-%dT%H:%M:%S',
  '%b %d %y %H:%M:%S GMT%z',
  '%b %d %y %H:%M:%S UTC%z',
  '%b %d %y %H:%M:%S',
  '%b %d %H:%M:%S GMT%z %Y',
  '%b %d %H:%M:%S UTC%z %Y',
  '%b %d %H:%M:%S %Y',
  '%a %b %d %H:%M:%S GMT%z %Y',
  '%a %b %d %H:%M:%S UTC%z %Y',
  '%a %b %d %H:%M:%S %Y',
}

function Date.parse(x, dont_offset)
  if type(x) == 'table' then return x end
  if type(x) == 'number' then return Date.strptime(tostring(x), '%s') end
  if type(x) ~= 'string' then return end
  local date, remainder
  for i = 1, #formats do
    date, remainder = Date.strptime(x, formats[i])
    if date then
      --if remainder then print('[' .. x:sub(-#remainder) .. ']') end
      if not remainder or x:sub(-#remainder) == remainder then
        -- compensate for GMT offset
        if date.gmtoff and not dont_offset then
          date = Date.parse(os.time(date) - date.gmtoff)
        end
        return date
      end
    end
  end
end

function Date.format(date, fmt)
  date = Date.parse(date)
  if date then
    return os.date(fmt, os.time(date))
  end
end

function Date.now()
  return os.time()
end

function Date.diff(d1, d2)
  d1 = Date.parse(d1)
  d2 = Date.parse(d2)
  if not d1 or not d2 then return end
  return os.time(d2) - os.time(d1)
end

function Date.add(date, delta)<commit_msg>more robust UTC offset calc<commit_after>local Date = require('./build/date')
local os = require('os')

local formats = {
  '%Y-%m-%d %H:%M:%S',
  '%a, %b %d %y %H:%M:%S',
  '%a, %b %d %Y %H:%M:%S',
  '%Y-%m-%dT%H:%M:%SZ%z',
  '%Y-%m-%dT%H:%M:%S',
  '%b %d %y %H:%M:%S',
  '%b %d %H:%M:%S %Y',
  '%a %b %d %H:%M:%S %Y',
  -- what else we missed?
}

function Date.parse(x, dont_offset)
  if type(x) == 'table' then return x end
  if type(x) == 'number' then return Date.strptime(tostring(x), '%s') end
  if type(x) ~= 'string' then return end
  local date, remainder
  for i = 1, #formats do
    date, remainder = Date.strptime(x, formats[i])
    if date then
      -- process remainder of form GMT+-hhmm, UTC+-hhmm
      if remainder then
        -- compose dummy date of simple format with the unparsed remainder
        remainder = '20120101000000' .. remainder:gsub('GMT', 'UTC')
        -- parse dummy date, to extract UTC offset
        local offset = Date.strptime(remainder, '%Y%m%d%H%M%S UTC%z')
        -- compensate the date for GMT offset, unless prohibited
        if not dont_offset then
          date = Date.parse(os.time(date) - offset.gmtoff)
        -- or just fill date.gmtoff field
        else
          date.gmtoff = offset.gmtoff
        end
      end
      return date
    end
  end
end

function Date.format(date, fmt)
  date = Date.parse(date)
  if date then
    return os.date(fmt, os.time(date))
  end
end

function Date.now()
  return os.time()
end

function Date.diff(d1, d2)
  d1 = Date.parse(d1)
  d2 = Date.parse(d2)
  if not d1 or not d2 then return end
  return os.time(d2) - os.time(d1)
end

function Date.add(date, delta)
{'fraction_non_alphanumeric': 0.20066085911685191, 'fraction_numerical': 0.010814058275758487, 'mean_word_length': 2.823191733639495, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5636048', 'n_tokens_mistral': 1404, 'n_tokens_neox': 1402, 'n_words': 397}