<commit_before>class CsvField < ActiveRecord::Base
  default_scope :order => 'sort_order'

  named_scope :morbidity_event_fields, :conditions => { :event_type => 'morbidity_event' }
  named_scope :morbidity_event_code_fields,
              :conditions => "event_type = 'morbidity_event' and use_code is not null"

  named_scope :place_event_fields, :conditions => { :event_type => 'place_event' }
  named_scope :place_event_code_fields,
              :conditions => "event_type = 'place_event' and use_code is not null"

  named_scope :contact_event_fields, :conditions => { :event_type => 'contact_event' }
  named_scope :contact_event_code_fields,
              :conditions => "event_type = 'contact_event' and use_code is not null"

  named_scope :lab_fields, :conditions => { :export_group => 'lab' }
  named_scope :lab_code_fields,
              :conditions => "export_group = 'lab' and use_code is not null"

  named_scope :treatment_fields, :conditions => { :export_group => 'treatment' }
  named_scope :treatment_code_fields,
              :conditions => "export_group = 'treatment' and use_code is not null"

  validates_length_of :short_name, :allow_nil => true, :maximum => 10

  def self.load_csv_fields(csv_fields)
    transaction do
      csv_fields.each do |k, v|
        CsvField.create!(v)
      end
    end
  end
end<commit_msg>need to be smarter about creating csv fields, so we avoid dupes<commit_after>class CsvField < ActiveRecord::Base
  default_scope :order => 'sort_order'

  named_scope :morbidity_event_fields, :conditions => { :event_type => 'morbidity_event' }
  named_scope :morbidity_event_code_fields,
              :conditions => "event_type = 'morbidity_event' and use_code is not null"

  named_scope :place_event_fields, :conditions => { :event_type => 'place_event' }
  named_scope :place_event_code_fields,
              :conditions => "event_type = 'place_event' and use_code is not null"

  named_scope :contact_event_fields, :conditions => { :event_type => 'contact_event' }
  named_scope :contact_event_code_fields,
              :conditions => "event_type = 'contact_event' and use_code is not null"

  named_scope :lab_fields, :conditions => { :export_group => 'lab' }
  named_scope :lab_code_fields,
              :conditions => "export_group = 'lab' and use_code is not null"

  named_scope :treatment_fields, :conditions => { :export_group => 'treatment' }
  named_scope :treatment_code_fields,
              :conditions => "export_group = 'treatment' and use_code is not null"

  validates_length_of :short_name, :allow_nil => true, :maximum => 10

  def self.load_csv_fields(csv_fields)
    transaction do
      csv_fields.each do |k, v|
        csv_field = CsvField.find_or_initialize_by_long_name(v)
        csv_field.save!
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.10171306209850108, 'fraction_numerical': 0.0014275517487508922, 'mean_word_length': 3.987544483985765, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7224748', 'n_tokens_mistral': 916, 'n_tokens_neox': 865, 'n_words': 212}