<commit_before>class GFS
  SERVER = 'http://www.ftp.ncep.noaa.gov/data/nccf/com/gfs/prod'
  RECORDS = {
    prate: 'PRATE:surface',
    tmp:   'TMP:2 m above ground',
    ugrd:  'UGRD:10 m above ground',
    vgrd:  'VGRD:10 m above ground',
    tcdc:  'TCDC:entire atmosphere'
  }
  FORECAST_HOURS = (3..114).step(3)

  def self.all
    Dir.foreach(Rails.root.join('tmp', 'gfs')).
      select { |dir| /\d{8}/ =~ dir }.
      sort_by { |dir| dir.to_i }.
      map { |dir| GFS.new(dir) }
  end

  def self.last
    self.all.last
  end

  def self.find(yyyymmddcc)
    if Dir.exist?(Rails.root.join('tmp', 'gfs', yyyymmddcc))
      self.new(yyyymmddcc)
    else
      nil
    end
  end

  def self.create(yyyymmddcc = "#{Time.now.strftime('%Y%m%d')}00")
    gfs = self.new(yyyymmddcc)
    gfs.download!
    gfs
  end

  def self.find_or_create(yyyymmddcc)
    self.find(yyyymmddcc) || self.create(yyyymmddcc)
  end

  def initialize(yyyymmddcc = "#{Time.now.strftime('%Y%m%d')}00")
    @yyyymmdd = yyyymmddcc[0..7]
    @cc = yyyymmddcc[8..9]
  end

  def path
    "/runs/#{@yyyymmdd}#{@cc}"
  end

  def time
    Time.parse("#{@yyyymmdd}#{@cc} +0000")
  end

  def read(field, hour: 3, latitude:, longitude:)
    key = [@yyyymmdd + @cc, field, hour, latitude, longitude].join(':')
    unless Redis.current.exists(key)
      wgrib2 = Rails.root.join('bin', 'wgrib2')
      filename = "gfs.t#{@cc}z.pgrb2f#{'%02d' % hour}"
      record = GFS::RECORDS[field]

      Dir.chdir(Rails.root.join('tmp', 'gfs', @yyyymmdd + @cc)) do |path|
        raise "'#{path}/#{filename}' not found" unless File.exists?(filename)
        out = `#{wgrib2} #{filename} -lon #{longitude} #{latitude} -match '#{record}'`
        lines = out.split("\n")
        fields = lines.first.split(':')
        params = Hash[*fields.last.split(',').map { |s| s.split('=') }.flatten]

        ttl = 1.hour
        val = params['val']
        Redis.current.setex(key, ttl, val)
      end
    end
    Redis.current.get(key).to_f
  end

  def download!
    curl = 'curl -O -f -s -S'
    FileUtils.mkpath(Rails.root.join('tmp', 'gfs', @yyyymmdd + @cc))
    Dir.chdir(Rails.root.join('tmp', 'gfs', @yyyymmdd + @cc)) do
      GFS::FORECAST_HOURS.map do |hour|
        filename = "gfs.t#{@cc}z.pgrb2f#{'%02d' % hour}"
        url = "#{GFS::SERVER}/gfs.#{@yyyymmdd}#{@cc}/#{filename}"

        Rails.logger.info("Downloading '#{url}.idx' ...")
        break unless system("#{curl} #{url}.idx")
        lines = IO.readlines("#{filename}.idx")
        n = lines.count
        ranges = lines.each_index.reduce([]) do |r, i|
          if GFS::RECORDS.values.any? { |record| lines[i].include?(record) }
            first = lines[i].split(':')[1].to_i
            last = ''

            j = i
            while (j += 1) < n<commit_msg>Remove chdir in GFS#read<commit_after>class GFS
  SERVER = 'http://www.ftp.ncep.noaa.gov/data/nccf/com/gfs/prod'
  RECORDS = {
    prate: 'PRATE:surface',
    tmp:   'TMP:2 m above ground',
    ugrd:  'UGRD:10 m above ground',
    vgrd:  'VGRD:10 m above ground',
    tcdc:  'TCDC:entire atmosphere'
  }
  FORECAST_HOURS = (3..114).step(3)

  def self.all
    Dir.foreach(Rails.root.join('tmp', 'gfs')).
      select { |dir| /\d{8}/ =~ dir }.
      sort_by { |dir| dir.to_i }.
      map { |dir| GFS.new(dir) }
  end

  def self.last
    self.all.last
  end

  def self.find(yyyymmddcc)
    if Dir.exist?(Rails.root.join('tmp', 'gfs', yyyymmddcc))
      self.new(yyyymmddcc)
    else
      nil
    end
  end

  def self.create(yyyymmddcc = "#{Time.now.strftime('%Y%m%d')}00")
    gfs = self.new(yyyymmddcc)
    gfs.download!
    gfs
  end

  def self.find_or_create(yyyymmddcc)
    self.find(yyyymmddcc) || self.create(yyyymmddcc)
  end

  def initialize(yyyymmddcc = "#{Time.now.strftime('%Y%m%d')}00")
    @yyyymmdd = yyyymmddcc[0..7]
    @cc = yyyymmddcc[8..9]
  end

  def path
    "/runs/#{@yyyymmdd}#{@cc}"
  end

  def time
    Time.parse("#{@yyyymmdd}#{@cc} +0000")
  end

  def read(field, hour: 3, latitude:, longitude:)
    key = [@yyyymmdd + @cc, field, hour, latitude, longitude].join(':')
    unless Redis.current.exists(key)
      wgrib2 = Rails.root.join('bin', 'wgrib2')
      filename = "gfs.t#{@cc}z.pgrb2f#{'%02d' % hour}"
      record = GFS::RECORDS[field]

      path = Rails.root.join('tmp', 'gfs', @yyyymmdd + @cc, filename)
      raise "'#{path}' not found" unless File.exists?(path)

      out = `#{wgrib2} #{path} -lon #{longitude} #{latitude} -match '#{record}'`
      lines = out.split("\n")
      fields = lines.first.split(':')
      params = Hash[*fields.last.split(',').map { |s| s.split('=') }.flatten]

      ttl = 1.hour
      val = params['val']
      Redis.current.setex(key, ttl, val)
    end
    Redis.current.get(key).to_f
  end

  def download!
    curl = 'curl -O -f -s -S'
    FileUtils.mkpath(Rails.root.join('tmp', 'gfs', @yyyymmdd + @cc))
    Dir.chdir(Rails.root.join('tmp', 'gfs', @yyyymmdd + @cc)) do
      GFS::FORECAST_HOURS.map do |hour|
        filename = "gfs.t#{@cc}z.pgrb2f#{'%02d' % hour}"
        url = "#{GFS::SERVER}/gfs.#{@yyyymmdd}#{@cc}/#{filename}"

        Rails.logger.info("Downloading '#{url}.idx' ...")
        break unless system("#{curl} #{url}.idx")
        lines = IO.readlines("#{filename}.idx")
        n = lines.count
        ranges = lines.each_index.reduce([]) do |r, i|
          if GFS::RECORDS.values.any? { |record| lines[i].include?(record) }
            first = lines[i].split(':')[1].to_i
            last = ''

            j = i
            while (j += 1) < n
{'fraction_non_alphanumeric': 0.19500723589001448, 'fraction_numerical': 0.013024602026049204, 'mean_word_length': 2.616088947024199, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1600614', 'n_tokens_mistral': 2370, 'n_tokens_neox': 2294, 'n_words': 407}