<commit_before># TEMPLATES
namespace :template do
  namespace :pp do

    $config        = Csible.get_config('api.json')
    output_dir     = 'transforms'
    templates_path = $config[:templates][:templates_path] ||= "templates"

    namespace :accessions do
      config_file     = "#{templates_path}/pastperfect/accessions/accessions.config.csv"
      mapping_file    = "#{templates_path}/pastperfect/accessions/accessions.map.csv"
      fields          = Csible::CSV.get_config(config_file)

      fields[:generate] = {
        ownertype: {
          from: :accessno,
          required: false,
          unique: false,
          process: ->(value) { 'person' }
        },
      }

      fields[:merge] = {
        descrip_: :notes_,
      }

      # rake template:pp:accessions:fields
      desc "Display fields for accessions objects"
      task :fields do |t|
        Csible::CSV.print_fields fields[:required], fields[:optional], fields[:generate].keys
      end

      # rake template:pp:accessions:process[templates/pastperfect/accessions/accessions.example.csv]
      desc "Create CollectionSpace accessions CSV from Past Perfect csv"
      task :process, [:csv, :output_dir] do |t, args|
        output_dir = args[:output_dir] || output_dir
        processor = Csible::CSV::PastPerfect.new(args[:csv], output_dir, mapping_file, fields)
        processor.process
      end
    end

    namespace :objects do
      config_file     = "#{templates_path}/pastperfect/objects/objects.config.csv"
      mapping_file    = "#{templates_path}/pastperfect/objects/objects.map.csv"
      fields          = Csible::CSV.get_config(config_file)

      # rake template:pp:objects:fields
      desc "Display fields for objects objects"
      task :fields do |t|
        Csible::CSV.print_fields fields[:required], fields[:optional], fields[:generate].keys
      end

      # rake template:pp:objects:process[templates/pastperfect/objects/objects.example.csv]
      desc "Create CollectionSpace collectionobjects CSV from Past Perfect csv"
      task :process, [:csv, :output_dir] do |t, args|
        output_dir = args[:output_dir] || output_dir
        processor = Csible::CSV::PastPerfect.new(args[:csv], output_dir, mapping_file, fields)
        processor.process
      end
    end
  end
end
<commit_msg>Additional pp field transforms<commit_after># TEMPLATES
namespace :template do
  namespace :pp do

    $config        = Csible.get_config('api.json')
    output_dir     = 'transforms'
    templates_path = $config[:templates][:templates_path] ||= "templates"

    namespace :accessions do
      config_file     = "#{templates_path}/pastperfect/accessions/accessions.config.csv"
      mapping_file    = "#{templates_path}/pastperfect/accessions/accessions.map.csv"
      fields          = Csible::CSV.get_config(config_file)

      fields[:generate] = {
        ownertype: {
          from: :accessno,
          required: false,
          unique: false,
          process: ->(value) { 'person' }
        },
      }

      fields[:merge] = {
        descrip_: :notes_,
      }

      fields[:transforms] = {
        accby: ->(value) { value.gsub(/^\W/, "").gsub(/\;/, " and ").squeeze(" ").strip },
        accdate: ->(value) { value.gsub(/^(\s*)-(\s*)-/, "") },
        recby: ->(value) { value.gsub(/^\W/, "").gsub(/\;/, " and ").squeeze(" ").strip },
        recfrom: ->(value) { value.gsub(/^\W/, "").gsub(/\;/, " and ").squeeze(" ").strip },
      }

      # rake template:pp:accessions:fields
      desc "Display fields for accessions objects"
      task :fields do |t|
        Csible::CSV.print_fields fields[:required], fields[:optional], fields[:generate].keys
      end

      # rake template:pp:accessions:process[templates/pastperfect/accessions/accessions.example.csv]
      desc "Create CollectionSpace accessions CSV from Past Perfect csv"
      task :process, [:csv, :output_dir] do |t, args|
        output_dir = args[:output_dir] || output_dir
        processor = Csible::CSV::PastPerfect.new(args[:csv], output_dir, mapping_file, fields)
        processor.process
      end
    end

    namespace :objects do
      config_file     = "#{templates_path}/pastperfect/objects/objects.config.csv"
      mapping_file    = "#{templates_path}/pastperfect/objects/objects.map.csv"
      fields          = Csible::CSV.get_config(config_file)

      # rake template:pp:objects:fields
      desc "Display fields for objects objects"
      task :fields do |t|
        Csible::CSV.print_fields fields[:required], fields[:optional], fields[:generate].keys
      end

      # rake template:pp:objects:process[templates/pastperfect/objects/objects.example.csv]
      desc "Create CollectionSpace collectionobjects CSV from Past Perfect csv"
      task :process, [:csv, :output_dir] do |t, args|
        output_dir = args[:output_dir] || output_dir
        processor = Csible::CSV::PastPerfect.new(args[:csv], output_dir, mapping_file, fields)
        processor.process
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.13067840704422654, 'fraction_numerical': 0.0, 'mean_word_length': 2.8624420401854715, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2093992', 'n_tokens_mistral': 1558, 'n_tokens_neox': 1473, 'n_words': 310}