<commit_before>require 'fastercsv'

# Include this in a controller
module CsvImport
  DEFAULT_PARSE_OPTS = { :headers => true, :header_converters => :symbol }

  private

  def each_csv_row(opts = {})
    @rows_imported = 0
    @bad_rows = []
    @unknown_headers = Set.new
    @imported = false

    if request.post?
      ActiveRecord::Base.transaction do
        ::Rails.logger.info("starting csv import")
        valid_headers = opts.delete(:valid_headers)

        FasterCSV.parse(params[:csv].read, DEFAULT_PARSE_OPTS.merge(opts)) do |row|
          begin
            row_data = row.to_hash
            if valid_headers
              row_data.delete_if {|header, value| ! valid_headers.include?(header) }
              @unknown_headers.merge(row.headers - row_data.keys)
            end
            obj = yield row_data
            obj.save! if obj && (obj.new_record? || obj.changed?)
            @rows_imported += 1
          rescue ActiveRecord::ActiveRecordError, ActiveRecord::UnknownAttributeError => exc
            row[:error] = exc<commit_msg>Ruby 1.9 compatibility<commit_after>if RUBY_VERSION =~ /1.8/
  require 'fastercsv'
else
  require 'csv'
end

# Include this in a controller
module CsvImport
  DEFAULT_PARSE_OPTS = { :headers => true, :header_converters => :symbol }

  private

  def each_csv_row(opts = {})
    @rows_imported = 0
    @bad_rows = []
    @unknown_headers = Set.new
    @imported = false

    if request.post?
      ActiveRecord::Base.transaction do
        ::Rails.logger.info("starting csv import")
        valid_headers = opts.delete(:valid_headers)

        (RUBY_VERSION =~ /1.8/ ?  FasterCSV : CSV).parse(params[:csv].read, DEFAULT_PARSE_OPTS.merge(opts)) do |row|
          begin
            row_data = row.to_hash
            if valid_headers
              row_data.delete_if {|header, value| ! valid_headers.include?(header) }
              @unknown_headers.merge(row.headers - row_data.keys)
            end
            obj = yield row_data
            obj.save! if obj && (obj.new_record? || obj.changed?)
            @rows_imported += 1
          rescue ActiveRecord::ActiveRecordError, ActiveRecord::UnknownAttributeError => exc
            row[:error] = exc
{'fraction_non_alphanumeric': 0.1081447963800905, 'fraction_numerical': 0.004524886877828055, 'mean_word_length': 2.5151033386327506, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2529835', 'n_tokens_mistral': 685, 'n_tokens_neox': 664, 'n_words': 141}