<commit_before>require_relative "caltrain_realtime"
require "sequel"
require "time"

def convert_times(time)
  now = DateTime.now
  # offset crap is to fix up time zones
  d = DateTime.parse(time).new_offset(now.offset) - now.offset

  diff = (d - now).to_f

  # if it's more than 12 hours ago or 12 hours from now, adjust
  if diff > 0.5
    d = d.prev
  elsif diff < -0.5
    d = d.next
  end

  if (d - now).abs > 0.5
    raise "tried to fix time #{time} at #{now} but got weird value #{d}"
  end

  d
end

def parse_arrival_time(reading_time, arr)
  m = /\A(\d+) min\.\z/.match(arr)

  raise "unable to parse arrival time '#{arr}'" if m.nil?

  reading_time + Rational(m[1].to_i, 1440)
end

def parse_scraped_times(deps)
  # in case there are no trains
  begin
    actual_time = deps.first.last.last.last<commit_msg>Fix DateTime next/prev function calls<commit_after>require_relative "caltrain_realtime"
require "sequel"
require "time"

def convert_times(time)
  now = DateTime.now
  # offset crap is to fix up time zones
  d = DateTime.parse(time).new_offset(now.offset) - now.offset

  diff = (d - now).to_f

  # if it's more than 12 hours ago or 12 hours from now, adjust
  if diff > 0.5
    d = d.prev_day
  elsif diff < -0.5
    d = d.next_day
  end

  if (d - now).abs > 0.5
    raise "tried to fix time #{time} at #{now} but got weird value #{d}"
  end

  d
end

def parse_arrival_time(reading_time, arr)
  m = /\A(\d+) min\.\z/.match(arr)

  raise "unable to parse arrival time '#{arr}'" if m.nil?

  reading_time + Rational(m[1].to_i, 1440)
end

def parse_scraped_times(deps)
  # in case there are no trains
  begin
    actual_time = deps.first.last.last.last
{'fraction_non_alphanumeric': 0.11901913875598086, 'fraction_numerical': 0.017942583732057416, 'mean_word_length': 3.1004901960784315, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1291386', 'n_tokens_mistral': 661, 'n_tokens_neox': 620, 'n_words': 199}