<commit_before>require_relative "../geocollider_parser"

class GeonamesParser
  extend GeocolliderParser

  def compare(names, places, filenames, csv_writer)
    filenames.each do |filename|
      geonames_data = File.open(filename, "rb")
      until geonames_data.eof()
        geonames_csv_string = geonames_data.readline.force_encoding('UTF-8').encode('UTF-8', :invalid => :replace)
        CSV.parse(geonames_csv_string, :headers => false, :col_sep => "\t", :quote_char => "\u{FFFF}") do |row|
          geoname = {}
          geoname["id"] = row[0]
          geoname["name"] = row[1]
          geoname["asciiname"] = row[2]
          geoname["alternatenames"] = row[3].nil? ? [] : row[3].split(',')
          geoname["latitude"] = row[4].to_f
          geoname["longitude"] = row[5].to_f
          geoname["featureclass"] = row[6]
          geoname["featurecode"] = row[7]
          geonames_names = ([geoname["name"], geoname["asciiname"]] + geoname["alternatenames"]).uniq.compact
          geonames_names.each do |name|
            $stderr.puts "Checking #{name}"
            if names.keys.include?(name)
              $stderr.puts "Name match, checking places..."
              names[name].each do |place|
                $stderr.puts "Checking #{place}"
                if self.class.check_point(places[place]['point'], Point.new(geoname["latitude"],geoname["longitude"]))
                  csv_writer << [place, geoname["id"]]
                end
              end
            end
          end
        end
      end
    end
  end
end<commit_msg>Use has_key? instead of keys.include<commit_after>require_relative "../geocollider_parser"

class GeonamesParser
  extend GeocolliderParser

  def compare(names, places, filenames, csv_writer)
    filenames.each do |filename|
      geonames_data = File.open(filename, "rb")
      until geonames_data.eof()
        geonames_csv_string = geonames_data.readline.force_encoding('UTF-8').encode('UTF-8', :invalid => :replace)
        CSV.parse(geonames_csv_string, :headers => false, :col_sep => "\t", :quote_char => "\u{FFFF}") do |row|
          geoname = {}
          geoname["id"] = row[0]
          geoname["name"] = row[1]
          geoname["asciiname"] = row[2]
          geoname["alternatenames"] = row[3].nil? ? [] : row[3].split(',')
          geoname["latitude"] = row[4].to_f
          geoname["longitude"] = row[5].to_f
          geoname["featureclass"] = row[6]
          geoname["featurecode"] = row[7]
          geonames_names = ([geoname["name"], geoname["asciiname"]] + geoname["alternatenames"]).uniq.compact
          geonames_names.each do |name|
            $stderr.puts "Checking #{name}"
            if names.has_key?(name)
              $stderr.puts "Name match, checking places..."
              names[name].each do |place|
                $stderr.puts "Checking #{place}"
                if self.class.check_point(places[place]['point'], Point.new(geoname["latitude"],geoname["longitude"]))
                  csv_writer << [place, geoname["id"]]
                end
              end
            end
          end
        end
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.14910600255427842, 'fraction_numerical': 0.0070242656449553, 'mean_word_length': 2.357984994640943, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1176525', 'n_tokens_mistral': 1024, 'n_tokens_neox': 985, 'n_words': 179}