<commit_before>require 'earth/locality/data_miner'
Industry.class_eval do
  def self.sic_lookup(naics_code)
    @sic_lookup ||= RemoteTable.new(:url => 'http://www.census.gov/epcd/naics02/NAICS02toSIC87.xls')
    record = @sic_lookup.find { |lookup| lookup['NAICS 2002'].to_i.to_s == naics_code }
    record ? record['SIC'].to_i.to_s : nil
  end

  data_miner do
    import "the U.S. Census 2002 NAICS code list",
           :url => 'http://www.census.gov/epcd/naics02/naicod02.txt',
           :skip => 4,
           :headers => false,
           :delimiter => '	' do
      key 'naics_code', :field_number => 0
      store 'sic', :synthesize => Proc.new { |row| Industry.sic_lookup(row[0]) }
      store 'description', :field_number => 1
    end
  end
end<commit_msg>Improved coding style<commit_after>require 'earth/locality/data_miner'
Industry.class_eval do
  def self.sic_lookup(naics_code)
    @sic_lookup ||= RemoteTable.new(:url => 'http://www.census.gov/epcd/naics02/NAICS02toSIC87.xls')
    if record = @sic_lookup.find { |lookup| lookup['NAICS 2002'].to_i.to_s == naics_code }
      record['SIC'].to_i.to_s
    end
  end

  data_miner do
    import "the U.S. Census 2002 NAICS code list",
           :url => 'http://www.census.gov/epcd/naics02/naicod02.txt',
           :skip => 4,
           :headers => false,
           :delimiter => '	' do
      key 'naics_code', :field_number => 0
      store 'sic', :synthesize => Proc.new { |row| Industry.sic_lookup(row[0]) }
      store 'description', :field_number => 1
    end
  end
end
{'fraction_non_alphanumeric': 0.14915693904020752, 'fraction_numerical': 0.028534370946822308, 'mean_word_length': 2.926208651399491, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4140473', 'n_tokens_mistral': 615, 'n_tokens_neox': 559, 'n_words': 109}