<commit_before><commit_msg>let there be geocode10m<commit_after>#! /usr/bin/env ruby


class GeoCode
  PI = Math.acos(-1.0)

  def self.checksum(lat_s, lon_s)
    c0 = 0
    c1 = 0
    lat_s.split('').each do |d|
      c1 += c0
      c0 += d.to_i(36)
      c0 = c0 % 31
    end
    c0 += 30
    lon_s.split('').each do |d|
      c1 += c0
      c0 += d.to_i(36)
      c0 = c0 % 31
    end
    c1 = c1 % 29
    c0.to_s(31) + c1.to_s(31)
  end


  def self.encode(latitude, longitude)
    lat = ((latitude + 90.0) * 10000).to_i
    lon = ((longitude + 180.0) * 10000 * Math.cos(PI * latitude / 180.0)).to_i
    lon_s = lon.to_s(36)
    lat_s = lat.to_s(36)
    cs = self.checksum(lat_s, lon_s)
    "#{lat_s}-#{cs}-#{lon_s}"
  end

  def self.decode(code)
    (lat_s, cs, lon_s) = code.split('-')
    raise 'invalid code' unless self.checksum(lat_s, lon_s) == cs
    lat = lat_s.to_i(36)
    latitude = (lat * 1e-4) - 90.0
    lon = lon_s.to_i(36)
    longitude = 180.0 + (lon * 1e-4 / Math.cos(PI * latitude / 180.0))
    longitude -= 360.0 if longitude > 180.0
    [latitude, longitude]
  end
end

if __FILE__ == $0
  code = ARGV[0]
  if /\A[0-9A-Za-z]+[-][0-9A-Za-z]{2}[-][0-9A-Za-z]+\z/ =~ code
    puts '%0.4f,%0.4f' % GeoCode.decode(code)
  elsif /\A-?\d+(\.\d*)?,-?\d+(\.\d*)?\z/ =~ code
    (lat, lon) = code.split(',')
    puts GeoCode.encode(lat.to_f, lon.to_f)
  else
    raise 'format error'
  end
end
{'fraction_non_alphanumeric': 0.19659332860184528, 'fraction_numerical': 0.07381121362668559, 'mean_word_length': 2.219178082191781, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '587056', 'n_tokens_mistral': 726, 'n_tokens_neox': 656, 'n_words': 133}