<commit_before>#
# Tasks to manage ingest of legacy Libra metadata
#

# pull in the helpers
require_dependency 'libra2/tasks/ingest_helpers'
include IngestHelpers

namespace :libra2 do

  namespace :ingest do

  desc "Enumerate new ingested items"
  task new_list: :environment do |t, args|

    count = 0
    GenericWork.search_in_batches( {} ) do |group|
      group.each do |gw_solr|

        begin
          gw = GenericWork.find( gw_solr['id'] )
          if gw.is_ingested_thesis?
            puts "#{gw.work_source} #{gw.identifier || 'None'}"
            count += 1
          end
        rescue => e
        end

      end

      puts "Listed #{count} new ingested work(s)"
    end
  end


  desc "Enumerate legacy ingested items"
  task legacy_list: :environment do |t, args|

    count = 0
    GenericWork.search_in_batches( {} ) do |group|
      group.each do |gw_solr|

        begin
           gw = GenericWork.find( gw_solr['id'] )
           if gw.is_legacy_thesis?
             puts "#{gw.work_source} #{gw.identifier || 'None'}"
             count += 1
           end
        rescue => e
        end

      end

      puts "Listed #{count} legacy ingested work(s)"
    end

  end

  desc "Purge new ingest ids; must provide the ingest directory"
  task purge_new_ingest_id: :environment do |t, args|

    ingest_dir = ARGV[ 1 ]
    if ingest_dir.nil?
      puts "ERROR: no ingest directory specified, aborting"
      next
    end
    task ingest_dir.to_sym do ; end

    # get the list of items to be ingested
    ingests = IngestHelpers.get_ingest_list( ingest_dir )
    if ingests.empty?
      puts "ERROR: ingest directory does not contain contains any items, aborting"
      next
    end

    count = 0
    ingests.each_with_index do | filename, ix |
      IngestHelpers.clear_ingest_id(File.join(ingest_dir, filename ) )
      count += 1
    end

    puts "Purged #{count} new ingest id(s)"
  end

  desc "Purge legacy ingest ids; must provide the ingest directory"
  task purge_legacy_ingest_id: :environment do |t, args|

    ingest_dir = ARGV[ 1 ]
    if ingest_dir.nil?
      puts "ERROR: no ingest directory specified, aborting"
      next
    end
    task ingest_dir.to_sym do ; end

    # get the list of items to be ingested
    ingests = IngestHelpers.get_legacy_ingest_list( ingest_dir )
    if ingests.empty?
      puts "ERROR: ingest directory does not contain contains any items, aborting"
      next
    end

    count = 0
    ingests.each_with_index do | dirname, ix |
      IngestHelpers.clear_legacy_ingest_id(File.join(ingest_dir, dirname ) )
      count += 1
    end

    puts "Purged #{count} legacy ingest id(s)"
  end

  desc "Finalize new ingest works; must provide the ingest directory"
  task finalize_new_ingests: :environment do |t, args|

    ingest_dir = ARGV[ 1 ]
    if ingest_dir.nil?
      puts "ERROR: no ingest directory specified, aborting"
      next
    end
    task ingest_dir.to_sym do ; end

    # get the list of items to be ingested
    ingests = IngestHelpers.get_ingest_list( ingest_dir )
    if ingests.empty?
      puts "ERROR: ingest directory does not contain contains any items, aborting"
      next
    end

    count = 0
    ingests.each_with_index do | filename, ix |
      work_id = IngestHelpers.get_ingest_id( File.join( ingest_dir, filename ) )

      if work_id.blank?
        puts "ERROR: no work id for #{filename}, continuing anyway"
        next
      end

      work = TaskHelpers.get_work_by_id( work_id )
      if work.nil?
        puts "ERROR: work #{work_id} does not exist, continuing anyway"
        next
      end

      # only finalize draft items...
      if work.is_draft?
         puts "Finalizing #{ix + 1} of #{ingests.length} (#{work_id})..."
         work.draft = 'false'

         if update_work_unassigned_doi( work ) == true
            count += 1
         end

      else
        puts "Work #{ix + 1} of #{ingests.length} (#{work_id}) already finalized, ignoring"
      end

    end

    puts "Finalized #{count} of #{ingests.length} ingest work(s)"
  end

  desc "Finalize legacy ingest works; must provide the ingest directory"
  task finalize_legacy_ingests: :environment do |t, args|
    puts "==> ERROR NOT IMPLEMENTED"<commit_msg>LIBRA-724: support for draft legacy items<commit_after>#
# Tasks to manage ingest of legacy Libra metadata
#

# pull in the helpers
require_dependency 'libra2/tasks/ingest_helpers'
include IngestHelpers

namespace :libra2 do

  namespace :ingest do

  desc "Enumerate new ingested items"
  task new_list: :environment do |t, args|

    count = 0
    GenericWork.search_in_batches( {} ) do |group|
      group.each do |gw_solr|

        begin
          gw = GenericWork.find( gw_solr['id'] )
          if gw.is_ingested_thesis?
            puts "#{gw.work_source} #{gw.identifier || 'None'}"
            count += 1
          end
        rescue => e
        end

      end

      puts "Listed #{count} new ingested work(s)"
    end
  end


  desc "Enumerate legacy ingested items"
  task legacy_list: :environment do |t, args|

    count = 0
    GenericWork.search_in_batches( {} ) do |group|
      group.each do |gw_solr|

        begin
           gw = GenericWork.find( gw_solr['id'] )
           if gw.is_legacy_thesis?
             puts "#{gw.work_source} #{gw.identifier || 'None'}"
             count += 1
           end
        rescue => e
        end

      end

      puts "Listed #{count} legacy ingested work(s)"
    end

  end

  desc "Purge new ingest ids; must provide the ingest directory"
  task purge_new_ingest_id: :environment do |t, args|

    ingest_dir = ARGV[ 1 ]
    if ingest_dir.nil?
      puts "ERROR: no ingest directory specified, aborting"
      next
    end
    task ingest_dir.to_sym do ; end

    # get the list of items to be ingested
    ingests = IngestHelpers.get_ingest_list( ingest_dir )
    if ingests.empty?
      puts "ERROR: ingest directory does not contain contains any items, aborting"
      next
    end

    count = 0
    ingests.each_with_index do | filename, ix |
      IngestHelpers.clear_ingest_id(File.join(ingest_dir, filename ) )
      count += 1
    end

    puts "Purged #{count} new ingest id(s)"
  end

  desc "Purge legacy ingest ids; must provide the ingest directory"
  task purge_legacy_ingest_id: :environment do |t, args|

    ingest_dir = ARGV[ 1 ]
    if ingest_dir.nil?
      puts "ERROR: no ingest directory specified, aborting"
      next
    end
    task ingest_dir.to_sym do ; end

    # get the list of items to be ingested
    ingests = IngestHelpers.get_legacy_ingest_list( ingest_dir )
    if ingests.empty?
      puts "ERROR: ingest directory does not contain contains any items, aborting"
      next
    end

    count = 0
    ingests.each_with_index do | dirname, ix |
      IngestHelpers.clear_legacy_ingest_id(File.join(ingest_dir, dirname ) )
      count += 1
    end

    puts "Purged #{count} legacy ingest id(s)"
  end

  desc "Finalize new ingest works; must provide the ingest directory"
  task finalize_new_ingests: :environment do |t, args|

    ingest_dir = ARGV[ 1 ]
    if ingest_dir.nil?
      puts "ERROR: no ingest directory specified, aborting"
      next
    end
    task ingest_dir.to_sym do ; end

    # get the list of items to be ingested
    ingests = IngestHelpers.get_ingest_list( ingest_dir )
    if ingests.empty?
      puts "ERROR: ingest directory does not contain contains any items, aborting"
      next
    end

    count = 0
    ingests.each_with_index do | filename, ix |
      work_id = IngestHelpers.get_ingest_id( File.join( ingest_dir, filename ) )

      if work_id.blank?
        puts "ERROR: no work id for #{filename}, continuing anyway"
        next
      end

      work = TaskHelpers.get_work_by_id( work_id )
      if work.nil?
        puts "ERROR: work #{work_id} does not exist, continuing anyway"
        next
      end

      # only finalize draft items...
      if work.is_draft?
         puts "Finalizing #{ix + 1} of #{ingests.length} (#{work_id})..."
         work.draft = 'false'

         if update_work_unassigned_doi( work ) == true
            count += 1
         end

      else
        puts "Work #{ix + 1} of #{ingests.length} (#{work_id}) already finalized, ignoring"
      end

    end

    puts "Finalized #{count} of #{ingests.length} ingest work(s)"
  end

  desc "Finalize legacy ingest works; must provide the ingest directory"
  task finalize_legacy_ingests: :environment do |t, args|

    ingest_dir = ARGV[ 1 ]
    if ingest_dir.nil?
      puts "ERROR: no ingest directory specified, aborting"
      next
    end
    task ingest_dir.to_sym do ; end

    # get the list of items to be ingested
    ingests = IngestHelpers.get_legacy_ingest_list( ingest_dir )
    if ingests.empty?
      puts "ERROR: ingest directory does not contain contains any items, aborting"
      next
    end

    count = 0
    ingests.each_with_index do | dirname, ix |
      work_id = IngestHelpers.get_legacy_ingest_id( File.join( ingest_dir, dirname ) )

      if work_id.blank?
        puts "ERROR: no work id for #{filename}, continuing anyway"
        next
      end

      work = TaskHelpers.get_work_by_id( work_id )
      if work.nil?
        puts "ERROR: work #{work_id} does not exist, continuing anyway"
        next
      end

      # only finalize draft items...
      if work.is_draft?
        puts "Finalizing #{ix + 1} of #{ingests.length} (#{work_id})..."
        work.draft = 'false'

        if update_work_unassigned_doi( work ) == true
          count += 1
        end

      else
        puts "Work #{ix + 1} of #{ingests.length} (#{work_id}) already finalized, ignoring"
      end

    end

    puts "Finalized #{count} of #{ingests.length} ingest work(s)"
{'fraction_non_alphanumeric': 0.08570255568100174, 'fraction_numerical': 0.00431078723185877, 'mean_word_length': 2.4213483146067416, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1989360', 'n_tokens_mistral': 3244, 'n_tokens_neox': 3164, 'n_words': 986}