<commit_before>require 'csv'
require 'street_address'
require 'cmess/guess_encoding'
require 'pry'

class AddressNormalizer

  def initialize(filename)
    @errors         = []
    @malformed_rows = []
    normalize_csv(filename)
  end

  # Given a CSV file, creates a new one with normalized addresses
  # If there are malformed rows/errors, saves those to a separate output file
  def normalize_csv(filename)

    # Overhead setup
    counter       = 0
    address_index = 0
    normalized_address_index = 0

    timestamp = Time.now.to_s.gsub(/:|-/,"").gsub(/\s+/,"_")

    normalized_output_file = File.open("#{File.basename(filename,".*")}_NormalizedAddresses_#{timestamp}.csv", "w")

    # Check encoding
    source_encoding = check_encoding(filename)

    puts "Normalizing addresses (this may take a while)..."
    
    IO.foreach(filename, :encoding => source_encoding) do |line|
      puts "On row #{counter}" if counter % 100 == 0

      # If the header row, get the index for the address and add a column for the address_normalized
      if counter == 0
        CSV.parse(line) do |row|
          address_index = row.index("address")
          row << "address_normalized"
          normalized_address_index = row.index("address_normalized")
          normalized_output_file.write(CSV.generate_line(row))
        end
        counter += 1

      # For normal rows, normalize, write to file, and catch errors
      else
        begin
          CSV.parse(line) do |row|
            begin
              sa = StreetAddress::US.parse(row[address_index] + ", , ")
            # Catch empty address
            rescue NoMethodError
              sa = ""
            end
            row << sa.to_s.upcase
            normalized_output_file.write(CSV.generate_line(row))
            counter += 1
          end
        # Rescue from problem rows
        rescue CSV::MalformedCSVError => er
          @errors << er.message
          @malformed_rows << line
        end
      end

    end

    # If there were problem rows, print them to the user and save them to a file
    unless @malformed_rows.empty?
      puts "Errors: #{@errors.to_s}\n\n\n" unless @errors.empty?
      malformed_row_output_file = File.open("AddressNormalizer_MalformedRows_#{timestamp}.txt", "w")
      puts "Malformed rows:"
      @malformed_rows.each do |mr|
        puts mr
        malformed_row_output_file.write(mr)
        malformed_row_output_file.write("\n")
      end
      malformed_row_output_file.close<commit_msg>moved 'overhead setup' into initialize'<commit_after>require 'csv'
require 'street_address'
require 'cmess/guess_encoding'
require 'pry'

class AddressNormalizer
  
  attr_reader :errors, :malformed_rows

  def initialize(filename)
    @errors         = []
    @malformed_rows = []
    @counter        = 0
    @address_index = 0
    @normalized_address_index = 0

    normalize_csv(filename)
  end

  # Given a CSV file, creates a new one with normalized addresses
  # If there are malformed rows/errors, saves those to a separate output file
  def normalize_csv(filename)

    timestamp = Time.now.to_s.gsub(/:|-/,"").gsub(/\s+/,"_")

    normalized_output_file = File.open("#{File.basename(filename,".*")}_NormalizedAddresses_#{timestamp}.csv", "w")

    # Check encoding
    source_encoding = check_encoding(filename)

    puts "Normalizing addresses (this may take a while)..."
    
    IO.foreach(filename, :encoding => source_encoding) do |line|
      puts "On row #{@counter}" if @counter % 100 == 0

      # If the header row, get the index for the address and add a column for the address_normalized
      if @counter == 0
        CSV.parse(line) do |row|
          @address_index = row.index("address")
          row << "address_normalized"
          @normalized_address_index = row.index("address_normalized")
          normalized_output_file.write(CSV.generate_line(row))
        end
        @counter += 1

      # For normal rows, normalize, write to file, and catch errors
      else
        begin
          CSV.parse(line) do |row|
            begin
              sa = StreetAddress::US.parse(row[@address_index] + ", , ")
            # Catch empty address
            rescue NoMethodError
              sa = ""
            end
            row << sa.to_s.upcase
            normalized_output_file.write(CSV.generate_line(row))
            @counter += 1
          end
        # Rescue from problem rows
        rescue CSV::MalformedCSVError => er
          @errors << er.message
          @malformed_rows << line
        end
      end

    end

    # If there were problem rows, print them to the user and save them to a file
    unless @malformed_rows.empty?
      puts "Errors: #{@errors.to_s}\n\n\n" unless @errors.empty?
      malformed_row_output_file = File.open("AddressNormalizer_MalformedRows_#{timestamp}.txt", "w")
      puts "Malformed rows:"
      @malformed_rows.each do |mr|
        puts mr
        malformed_row_output_file.write(mr)
        malformed_row_output_file.write("\n")
      end
      malformed_row_output_file.close
{'fraction_non_alphanumeric': 0.09930761622156281, 'fraction_numerical': 0.003956478733926805, 'mean_word_length': 2.4559125085440874, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1509700', 'n_tokens_mistral': 1517, 'n_tokens_neox': 1482, 'n_words': 394}