<commit_before>module Import
  extend ActiveSupport::Autoload
  autoload :DummyUser
  autoload :MonographBuilder
  autoload :SectionBuilder
  autoload :FileSetBuilder
  autoload :Importer
  autoload :CSVParser
  autoload :RowData

  # something to note is that the multivalued :yes/:no values mirror the model, such that assignment will work properly as a scalar or array...
  # ... the third :yes_split means that this is a field we actually want to *use* as multivalued, and so will split the CSV field on semicolons to do so

  UNIVERSAL_FIELDS =
    [
      { field_name: 'Title', metadata_name: 'title', required: true, multivalued: :yes_split },
      { field_name: 'Primary Creator <NAME>', metadata_name: 'creator_family_name', required: true, multivalued: :no },
      { field_name: 'Primary Creator First Name', metadata_name: 'creator_given_name', required: true, multivalued: :no },
      { field_name: 'Description', metadata_name: 'description', required: false, multivalued: :yes }
    ].freeze

  MONOGRAPH_FIELDS =
    [
      { field_name: 'Publisher', metadata_name: 'publisher', multivalued: :yes },
      { field_name: 'Subject', metadata_name: 'subject', multivalued: :yes },
      { field_name: 'ISBN (hardcover)', metadata_name: 'isbn', multivalued: :yes },
      { field_name: 'ISBN (paper)', metadata_name: 'isbn_paper', multivalued: :yes },
      { field_name: 'ISBN (ebook)', metadata_name: 'isbn_ebook', multivalued: :yes },
      { field_name: 'Buy Book URL', metadata_name: 'buy_url', multivalued: :yes }
    ].freeze

  ASSET_FIELDS =
    [
      # for now leave File Name separate, but eventually when external resources are figured out it should go in here too with required => false
      # { field_name: 'File Name', metadata_name: 'title', required: true, multivalued: :no },
      { field_name: 'Resource Type', metadata_name: 'resource_type', required: true, multivalued: :yes, acceptable_values: ['audio', 'image', 'dataset', 'table', '3D model', 'text', 'video'] },
      { field_name: 'Externally Hosted Resource', metadata_name: 'external_resource', required: true, multivalued: :no, acceptable_values: ['yes', 'no'] },<commit_msg>no errors for missing creator names<commit_after>module Import
  extend ActiveSupport::Autoload
  autoload :DummyUser
  autoload :MonographBuilder
  autoload :SectionBuilder
  autoload :FileSetBuilder
  autoload :Importer
  autoload :CSVParser
  autoload :RowData

  # something to note is that the multivalued :yes/:no values mirror the model, such that assignment will work properly as a scalar or array...
  # ... the third :yes_split means that this is a field we actually want to *use* as multivalued, and so will split the CSV field on semicolons to do so

  UNIVERSAL_FIELDS =
    [
      { field_name: 'Title', metadata_name: 'title', required: true, multivalued: :yes_split },
      { field_name: 'Primary Creator Last Name', metadata_name: 'creator_family_name', required: false, multivalued: :no },
      { field_name: 'Primary Creator First Name', metadata_name: 'creator_given_name', required: false, multivalued: :no },
      { field_name: 'Description', metadata_name: 'description', required: false, multivalued: :yes }
    ].freeze

  MONOGRAPH_FIELDS =
    [
      { field_name: 'Publisher', metadata_name: 'publisher', multivalued: :yes },
      { field_name: 'Subject', metadata_name: 'subject', multivalued: :yes },
      { field_name: 'ISBN (hardcover)', metadata_name: 'isbn', multivalued: :yes },
      { field_name: 'ISBN (paper)', metadata_name: 'isbn_paper', multivalued: :yes },
      { field_name: 'ISBN (ebook)', metadata_name: 'isbn_ebook', multivalued: :yes },
      { field_name: 'Buy Book URL', metadata_name: 'buy_url', multivalued: :yes }
    ].freeze

  ASSET_FIELDS =
    [
      # for now leave File Name separate, but eventually when external resources are figured out it should go in here too with required => false
      # { field_name: 'File Name', metadata_name: 'title', required: true, multivalued: :no },
      { field_name: 'Resource Type', metadata_name: 'resource_type', required: true, multivalued: :yes, acceptable_values: ['audio', 'image', 'dataset', 'table', '3D model', 'text', 'video'] },
      { field_name: 'Externally Hosted Resource', metadata_name: 'external_resource', required: true, multivalued: :no, acceptable_values: ['yes', 'no'] },
{'fraction_non_alphanumeric': 0.12068571428571429, 'fraction_numerical': 0.00045714285714285713, 'mean_word_length': 4.490589711417817, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1621582', 'n_tokens_mistral': 1358, 'n_tokens_neox': 1328, 'n_words': 436}