<commit_before>module Geonames
  class Dump
    URL = "http://download.geonames.org/export/"
    TMP = "/tmp/geonames/"

    def initialize(codes, kind)
      @codes = codes
      @kind = kind
      if codes.respond_to? :each
        for code in codes
          info "\nWorking on #{kind} for #{code}"
          file = get_file(code)
          download file
          uncompress file unless code == "country"
          parse file
        end
      end

    end

    def get_file(code)
      code == "country" ? "countryInfo.txt" : "#{code.upcase}.zip"
    end

    def download(file)
      Dir.mkdir(TMP) unless File.exists?(TMP)
      Dir.mkdir(TMP + @kind.to_s) unless File.exists?(TMP + @kind.to_s)
      fname = TMP + "#{@kind}/#{file}"
      return if File.exists?(fname)
      `curl #{URL}/#{@kind}/#{file} -o #{fname}`
    end

    def uncompress(file)
      info "Uncompressing #{file}"
      `unzip -quo /tmp/geonames/#{@kind}/#{file} -d /tmp/geonames/#{@kind}`
    end

    def parse_line(l)
      return if l =~ /^#|^iso/i
      if @kind == :dump
        if l =~ /^\D/
          Country.parse(l)
        else
          if Opt[:level] != "all"
            return unless l =~ /ADM\d/ # ADM2 => cities
          end
        end
      end
      Spot.new(l, @kind)
    end

    def parse(file)
      red = 0
      start = Time.now
      File.open("/tmp/geonames/#{@kind}/#{file.gsub("zip", "txt")}") do |f|
        while line = f.gets
          if record = parse_line(line)
            Cache[@kind] << record
            red += 1
          end
        end
        total = Time.now - start
        info "#{red} #{@kind} entries parsed in #{total} sec (#{(red/total).to_i}/s)"
      end
      rescue Errno::ENOENT => e
      info "Failed to download #{file}, skipping."
    end
<commit_msg>Changed some lines to allow the storage of countries.<commit_after>module Geonames
  class Dump
    URL = "http://download.geonames.org/export/"
    TMP = "/tmp/geonames/"

    def initialize(codes, kind)
      @codes = codes
      @kind = kind
      if codes.respond_to? :each
        for code in codes
          info "\nWorking on #{kind} for #{code}"
          file = get_file(code)
          download file
          uncompress file unless code == "country"
          parse file
        end
      end

    end

    def get_file(code)
      code == "country" ? "countryInfo.txt" : "#{code.upcase}.zip"
    end

    def download(file)
      Dir.mkdir(TMP) unless File.exists?(TMP)
      Dir.mkdir(TMP + @kind.to_s) unless File.exists?(TMP + @kind.to_s)
      fname = TMP + "#{@kind}/#{file}"
      return if File.exists?(fname)
      `curl #{URL}/#{@kind}/#{file} -o #{fname}`
    end

    def uncompress(file)
      info "Uncompressing #{file}"
      `unzip -quo /tmp/geonames/#{@kind}/#{file} -d /tmp/geonames/#{@kind}`
    end

    def parse_line(l)
      return if l =~ /^#|^iso/i
      if @kind == :dump
        if l =~ /^\D/
          return Country.parse(l)
        else
          if Opt[:level] != "all"
            return unless l =~ /ADM\d/ # ADM2 => cities
          end
        end
      end
      Spot.new(l, @kind)
    end

    def parse(file)
      red = 0
      start = Time.now
      File.open("/tmp/geonames/#{@kind}/#{file.gsub("zip", "txt")}") do |f|
        while line = f.gets
          if record = parse_line(line)
            Cache[@kind] << record
            red += 1
          end
        end
        total = Time.now - start
        info "#{red} #{@kind} entries parsed in #{total} sec (#{(red/total).to_i}/s)"
      end
      rescue Errno::ENOENT => e
      info "Failed to download #{file}, skipping."
    end

{'fraction_non_alphanumeric': 0.14883081155433286, 'fraction_numerical': 0.0016506189821182944, 'mean_word_length': 1.7862068965517242, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4645792', 'n_tokens_mistral': 1212, 'n_tokens_neox': 1211, 'n_words': 319}