<commit_before># Parser for loading FixMyStreet contacts data

class Parsers::FmsContactsParser 

  def csv_options
    { :quote_char => '"', 
      :col_sep => ",", 
      :row_sep =>:auto, 
      :return_headers => false,
      :headers => :first_row,
      :encoding => 'N' }
  end
  
  def parse_contacts filepath
    csv_data = File.read(filepath)
    FasterCSV.parse(csv_data, csv_options) do |row|
      next if row['deleted'] == 't'
      confirmed = !row['confirmed'].nil?
      yield CouncilContact.new({:area_id => row['area_id'], 
                                :email => row['email'], 
                                :confirmed => confirmed,
                                :category => row['category'],
                                :notes => row['note']})
    end
  end
  
end<commit_msg>Handle special case for Essex council<commit_after># Parser for loading FixMyStreet contacts data

class Parsers::FmsContactsParser

  def csv_options
    { :quote_char => '"',
      :col_sep => ",",
      :row_sep =>:auto,
      :return_headers => false,
      :headers => :first_row,
      :encoding => 'N' }
  end

  def parse_contacts filepath
    csv_data = File.read(filepath)
    FasterCSV.parse(csv_data, csv_options) do |row|
      next if row['deleted'] == 't'
      next if row['category'].blank?
      confirmed = !row['confirmed'].nil?
      if row['area_id'] == '2225' and row['email'].strip == 'SPECIAL'
        district_mappings = { 'eastarea' => [2315, 2312],
                              'midarea' => [2317, 2314, 2316],
                              'southarea' => [2319, 2320, 2310],
                              'westarea' => [2309, 2311, 2318, 2313] }
        district_mappings.each do |area_name, district_ids|
          district_ids.each do |district_id|
            yield CouncilContact.new({:area_id => row['area_id'],
                                      :email => "<EMAIL>",
                                      :confirmed => confirmed,
                                      :district_id => district_id,
                                      :category => row['category'].strip,
                                      :notes => row['note']})
          end
        end
      else
        yield CouncilContact.new({:area_id => row['area_id'],
                                  :email => row['email'].strip,
                                  :confirmed => confirmed,
                                  :category => row['category'].strip,
                                  :notes => row['note']})
      end
    end
  end

end
{'fraction_non_alphanumeric': 0.12866640594446618, 'fraction_numerical': 0.02033633163863903, 'mean_word_length': 1.5351833498513379, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5430608', 'n_tokens_mistral': 772, 'n_tokens_neox': 698, 'n_words': 156}