<commit_before>class District < ActiveRecord::Base

  def self.sort(fields)
    cols = ['blacks', 'whites', 'hispanics', 'asians', 'american_indians', 'males', 'females', 'unemployment', 'median_age', 'median_house_value', 'median_rent', 'median_household_income']
    fields.sort {|a, b| cols.index(a) <=> cols.index(b)}
  end
  
  def self.data_for(legislators, columns)
    data = {}
    
    # only use columns that were checked
    columns.each {|column, use| data[column] = {} if use == '1'}
    
    legislators.each do |legislator|
      leg_district = ['Senior Seat', 'Junior Seat'].include?(legislator.district) ? 'state' : legislator.district
      if district = District.find_by_state_and_district(legislator.state, leg_district)
        data.keys.each do |column|
          data[column][legislator.bioguide_id] = district.send(column)
        end
      end
    end
    data
  end
  
  def self.update(options = {})
    # preparations
    data_dir = "data/census/2000"
    old_dir = Dir.pwd
    
    if options[:download] and File.exists?(data_dir)
      FileUtils.rm_rf data_dir
    end
    
    if !File.exists?(data_dir)
      FileUtils.mkdir_p data_dir
      FileUtils.chdir data_dir
      download_census
      unzip_census
    else
      FileUtils.chdir data_dir
    end
    
    state_counts = {}
    
    states.keys.sort.each do |state|
      puts "[#{state}] Reading in pages..."
      # read in all the CSV (oh boy)
      pages = self.district_pages state
      
      ##### per-district files #####
      
      # Figure out the map of districts to rows for this state
      
      header = File.new "sl1/#{state}/sl500-in-sl040-#{state.downcase}geo.h10"
      
      # constructed in format "1" => "20" for "District 1 is at row index 20"
      districts = {}
      
      i = 0
      while !header.eof?
        row = header.readline
        name = row[200..289]
        if district = district_for(name) and districts[district].nil?
          districts[district] = i
        end
        i += 1
      end
      puts "[#{state}} Found #{districts.keys.size} districts..."
      
      # Loop through each region name, making a district for each congressional district
      district_count = 0
      districts.each do |name, i|
        puts "  [District #{name}] Parsing census data..."
        
        district = District.find_or_initialize_by_state_and_district state, name
        fill_district district, pages, i
        district.save!
         
        district_count += 1
      end
      <commit_msg>Census data is loadable in the table<commit_after>class District < ActiveRecord::Base

  def self.sort(fields)
    cols = ['blacks', 'whites', 'hispanics', 'asians', 'american_indians', 'males', 'females', 'unemployment', 'median_age', 'median_house_value', 'median_rent', 'median_household_income']
    fields.sort {|a, b| cols.index(a) <=> cols.index(b)}
  end
  
  def self.data_for(legislators, columns)
    data = {}
    
    # only use columns that were checked
    columns.each {|column, use| data[column] = {} if use == '1'}
    
    legislators.each do |legislator|
      leg_district = ['Senior Seat', 'Junior Seat'].include?(legislator.district) ? 'state' : legislator.district
      if district = District.find_by_state_and_district(legislator.state, leg_district)
        data.keys.each do |column|
          data[column][legislator.bioguide_id] = district.send(column)
        end
      end
    end
    data
  end
  
  def self.update(options = {})
    # preparations
    data_dir = "data/census/2000"
    old_dir = Dir.pwd
    
    if options[:download] and File.exists?(data_dir)
      FileUtils.rm_rf data_dir
    end
    
    if !File.exists?(data_dir)
      FileUtils.mkdir_p data_dir
      FileUtils.chdir data_dir
      download_census
      unzip_census
    else
      FileUtils.chdir data_dir
    end
    
    state_counts = {}
    
    states.keys.sort.each do |state|
      puts "[#{state}] Reading in pages..."
      # read in all the CSV (oh boy)
      pages = self.district_pages state
      
      ##### per-district files #####
      
      # Figure out the map of districts to rows for this state
      
      header = File.new "districts/sl1/#{state}/sl500-in-sl040-#{state.downcase}geo.h10"
      
      # constructed in format "1" => "20" for "District 1 is at row index 20"
      districts = {}
      
      i = 0
      while !header.eof?
        row = header.readline
        name = row[200..289]
        if district = district_for(name) and districts[district].nil?
          districts[district] = i
        end
        i += 1
      end
      puts "[#{state}} Found #{districts.keys.size} districts..."
      
      # Loop through each region name, making a district for each congressional district
      district_count = 0
      districts.each do |name, i|
        puts "  [District #{name}] Parsing census data..."
        
        district = District.find_or_initialize_by_state_and_district state, name
        fill_district district, pages, i
        district.save!
         
        district_count += 1
      end
      
{'fraction_non_alphanumeric': 0.11137301119622864, 'fraction_numerical': 0.011785503830288745, 'mean_word_length': 2.3632760898282696, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7204403', 'n_tokens_mistral': 1669, 'n_tokens_neox': 1603, 'n_words': 406}