<commit_before>class Address
  require 'csv'
  
  @@preferred_regions = []
  CSV.foreach("public/preferred/regions.csv") do |row|
    @@preferred_regions << Gnlookup::Region.find_by_geoname_id(row[0])
  end
  @@preferred_cities = []
  CSV.foreach("public/preferred/cities.csv") do |row|
    @@preferred_cities << Gnlookup::City.find_by_geoname_id(row[0])
  end
  
  attr_accessor :string
  
  def initialize(string)
    @string = string
  end
  
  # Cities
  # Returns an array of city objects
  def cities
    cities = []
    pieces = self.string.split(",")
    pieces.each do |piece|
      value = piece.strip.chomp(".")
      cities << Gnlookup::City.where("name_n = ?", value)
    end
    return cities.flatten
  end
  def lone_cities
    return self.cities - self.city_region_combos - self.city_country_combos - self.city_region_country_combos
  end
  def preferred_lone_cities
    return self.lone_cities.reject{|c| !@@preferred_cities.include?(c)}
  end
  
  # Regions
  # Returns an array of regions objects
  def regions
    regions = []
    pieces = self.string.split(",")
    pieces.each do |piece|
      value = piece.strip.chomp(".")
      us_address_pattern = /(^[A-Z]{2}\s\d{5}$)|(^[A-Z]{2}\s\d{5}-\d{4}$)/
      if !(us_address_pattern =~ value).nil? # If this matches a US address pattern
        state_name = value.split(" ")[0].chomp(".") # Get the state name
        regions << Gnlookup::Region.where("name_n = ? OR iso_n = ?", state_name, state_name) # Search with state names
      else
        regions << Gnlookup::Region.where("name_n = ? OR iso_n = ?", value, value) # Search with raw value
      end
    end
    return regions.flatten
  end
  def lone_regions
    return self.regions - self.city_region_combos.collect{|c| c.region} - self.city_region_country_combos.collect{|c| c.region} - self.region_country_combos
  end
  def preferred_lone_regions
    return self.lone_regions.reject{|r| !@@preferred_regions.include?(r)}
  end
  
  # Zipcodes
  # Returns an array of zipcode objects
  def zipcodes
    zipcodes = []
    pieces = self.string.split(",")
    pieces.each do |piece|
      value = piece.strip.chomp(".")
      us_address_pattern = /(^[A-Z]{2}\s\d{5}$)|(^[A-Z]{2}\s\d{5}-\d{4}$)/
      if !(us_address_pattern =~ value).nil? # If this matches a US address pattern
        zipcode = value.split(" ")[1].split("-")[0]
        zipcodes << Gnlookup::Zipcode.where("zipcode_n = ?", zipcode)
      end
    end
    return zipcodes.flatten
  end
  
  # Countries
  # Returns an array of country objects
  def countries
    countries = []
    pieces = self.string.split(",")
    pieces.each do |piece|
      value = piece.strip.chomp(".")
      countries << Gnlookup::Country.where("name_n = ? OR iso_n = ? OR iso3_n = ?", value, value, value)
    end
    return countries.flatten
  end
  def lone_countries
    return self.countries - self.city_country_combos.collect{|c| c.country} - self.city_region_country_combos.collect{|c| c.country} - self.region_country_combos.collect{|r| r.country}
  end
  
  # City_region_country combos
  # Returns an array of city objects
  def city_region_country_combos
    combos = []
    if !self.cities.blank? && !self.regions.blank? && !self.countries.blank?
      self.cities.each do |city|
        if self.regions.include?(city.region) && self.countries.include?(city.country)
          combos << city
        end
      end
    end
    return combos
  end
  
  # City_region combos
  # Returns an array of city objects
  def city_region_combos
    combos = []
    if !self.cities.blank? && !self.regions.blank?
      self.cities.each do |city|
        if self.regions.include? city.region<commit_msg>Fixed geo lookup query.<commit_after>class Address
  require 'csv'
  
  @@preferred_regions = []
  CSV.foreach("public/preferred/regions.csv") do |row|
    @@preferred_regions << Gnlookup::Region.find_by_geoname_id(row[0])
  end
  @@preferred_cities = []
  CSV.foreach("public/preferred/cities.csv") do |row|
    @@preferred_cities << Gnlookup::City.find_by_geoname_id(row[0])
  end
  
  attr_accessor :string
  
  def initialize(string)
    @string = string
  end
  
  # Cities
  # Returns an array of city objects
  def cities
    cities = []
    pieces = self.string.split(",")
    pieces.each do |piece|
      value = piece.strip.chomp(".")
      cities << Gnlookup::City.where("name_n = ?", I18n.transliterate(value).downcase)
    end
    return cities.flatten
  end
  def lone_cities
    return self.cities - self.city_region_combos - self.city_country_combos - self.city_region_country_combos
  end
  def preferred_lone_cities
    return self.lone_cities.reject{|c| !@@preferred_cities.include?(c)}
  end
  
  # Regions
  # Returns an array of regions objects
  def regions
    regions = []
    pieces = self.string.split(",")
    pieces.each do |piece|
      value = piece.strip.chomp(".")
      us_address_pattern = /(^[A-Z]{2}\s\d{5}$)|(^[A-Z]{2}\s\d{5}-\d{4}$)/
      if !(us_address_pattern =~ value).nil? # If this matches a US address pattern
        state_name = value.split(" ")[0].chomp(".") # Get the state name
        regions << Gnlookup::Region.where("name_n = ? OR iso_n = ?", I18n.transliterate(state_name).downcase, I18n.transliterate(state_name).downcase) # Search with state names
      else
        regions << Gnlookup::Region.where("name_n = ? OR iso_n = ?", I18n.transliterate(value).downcase, I18n.transliterate(value).downcase) # Search with raw value
      end
    end
    return regions.flatten
  end
  def lone_regions
    return self.regions - self.city_region_combos.collect{|c| c.region} - self.city_region_country_combos.collect{|c| c.region} - self.region_country_combos
  end
  def preferred_lone_regions
    return self.lone_regions.reject{|r| !@@preferred_regions.include?(r)}
  end
  
  # Zipcodes
  # Returns an array of zipcode objects
  def zipcodes
    zipcodes = []
    pieces = self.string.split(",")
    pieces.each do |piece|
      value = piece.strip.chomp(".")
      us_address_pattern = /(^[A-Z]{2}\s\d{5}$)|(^[A-Z]{2}\s\d{5}-\d{4}$)/
      if !(us_address_pattern =~ value).nil? # If this matches a US address pattern
        zipcode = value.split(" ")[1].split("-")[0]
        zipcodes << Gnlookup::Zipcode.where("zipcode_n = ?", I18n.transliterate(zipcode).downcase)
      end
    end
    return zipcodes.flatten
  end
  
  # Countries
  # Returns an array of country objects
  def countries
    countries = []
    pieces = self.string.split(",")
    pieces.each do |piece|
      value = piece.strip.chomp(".")
      countries << Gnlookup::Country.where("name_n = ? OR iso_n = ? OR iso3_n = ?", I18n.transliterate(value).downcase, I18n.transliterate(value).downcase, I18n.transliterate(value).downcase)
    end
    return countries.flatten
  end
  def lone_countries
    return self.countries - self.city_country_combos.collect{|c| c.country} - self.city_region_country_combos.collect{|c| c.country} - self.region_country_combos.collect{|r| r.country}
  end
  
  # City_region_country combos
  # Returns an array of city objects
  def city_region_country_combos
    combos = []
    if !self.cities.blank? && !self.regions.blank? && !self.countries.blank?
      self.cities.each do |city|
        if self.regions.include?(city.region) && self.countries.include?(city.country)
          combos << city
        end
      end
    end
    return combos
  end
  
  # City_region combos
  # Returns an array of city objects
  def city_region_combos
    combos = []
    if !self.cities.blank? && !self.regions.blank?
      self.cities.each do |city|
        if self.regions.include? city.region
{'fraction_non_alphanumeric': 0.1340774786605384, 'fraction_numerical': 0.006565988181221274, 'mean_word_length': 3.145890038105607, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3811561', 'n_tokens_mistral': 2677, 'n_tokens_neox': 2608, 'n_words': 544}