<commit_before># Processes an incremental dump file for SCSB coming from Alma and updates the
# DumpFile for submission to the S3 Bucket.
class RecapDumpFileProcessingJob < ActiveJob::Base
  # def perform(dump_file, skip_boundwiths=true)
  def perform(dump_file)
    # Unzip/Parse dump file
    records = extract_records(dump_file.path)
    # Save the file.
    write_records(records, dump_file.path)
    # Transfer it to S3.
    RecapTransferJob.perform_later(dump_file)
  end

  # @param path [String] Path on disk to the records file.
  # @return Hash<String, Array<MARC::Record>> Hash of filenames to the
  #   MARC::Records that they represent.
  def extract_records(path)
    # TODO: skip it if it's a host or constituent record
    tar_extract = Gem::Package::TarReader.new(Zlib::GzipReader.open(path))
    tar_extract.tap(&:rewind)
    Hash[
      tar_extract.map do |tar_entry|
        content = StringIO.new(tar_entry.read)
        records = MARC::XMLReader.new(content, external_encoding: 'UTF-8').to_a.map do |record|
          # ScsbDumpRecord will handle converting from the dump's MARC-XML to
          # the proper format for submitCollection
          AlmaAdapter::ScsbDumpRecord.new(marc_record: record)
        end
        [tar_entry.header.name, records]
      end
    ]
  end

  # Persist the mutated records back to the file.
  # @param path [String] Path on disk to the records file.
  # @param records [Hash<String, Array<AlmaAdapter::ScsbDumpRecord>>] Hash of filenames to the
  #   MARC::Records that they represent.
  def write_records(records, path)
    records_with_content = records.map do |filename, file_records|
      content = StringIO.new
      writer = MARC::XMLWriter.new(content)
      file_records.each do |record|
        # TODO: pass a host record? if it's a constituent?
        writer.write(record.transformed_record)
      end
      writer.close
      [filename, content.string]
    end
    archive_path(path, records_with_content)
  end

  def archive_path(path, records_with_content)
    File.open(path, "wb") do |file|<commit_msg>Skip or extract boundwith records<commit_after># Processes an incremental dump file for SCSB coming from Alma and updates the
# DumpFile for submission to the S3 Bucket.
class RecapDumpFileProcessingJob < ActiveJob::Base
  def perform(dump_file)
    # Unzip/Parse dump file
    records = extract_records(dump_file.path)
    # Save the file.
    write_records(records, dump_file.path)
    # Transfer it to S3.
    RecapTransferJob.perform_later(dump_file)
  end

  # @param path [String] Path on disk to the records file.
  # @return Hash<String, Array<MARC::Record>> Hash of filenames to the
  #   MARC::Records that they represent.
  def extract_records(path, boundwiths: false)
    tar_extract = Gem::Package::TarReader.new(Zlib::GzipReader.open(path))
    tar_extract.tap(&:rewind)
    Hash[
      tar_extract.map do |tar_entry|
        content = StringIO.new(tar_entry.read)
        records = MARC::XMLReader.new(content, external_encoding: 'UTF-8').to_a.map do |record|
          # ScsbDumpRecord will handle converting from the dump's MARC-XML to
          # the proper format for submitCollection
          scsb_record = AlmaAdapter::ScsbDumpRecord.new(marc_record: record)
          # If boundwith param is true; return record if it is a boundwith or
          # skip record if it is not a boundwith.
          # If boundwith param is false; return record if it is not a boundwith
          # or skip record if it is a boundwith.
          if boundwiths && scsb_record.boundwith?
            scsb_record
          elsif boundwiths && !scsb_record.boundwith?
            next
          elsif scsb_record.boundwith?
            next
          else
            scsb_record
          end
        end
        [tar_entry.header.name, records]
      end
    ]
  end

  # Persist the mutated records back to the file.
  # @param path [String] Path on disk to the records file.
  # @param records [Hash<String, Array<AlmaAdapter::ScsbDumpRecord>>] Hash of filenames to the
  #   MARC::Records that they represent.
  def write_records(records, path)
    records_with_content = records.map do |filename, file_records|
      content = StringIO.new
      writer = MARC::XMLWriter.new(content)
      file_records.each do |record|
        # TODO: pass a host record? if it's a constituent?
        writer.write(record.transformed_record)
      end
      writer.close
      [filename, content.string]
    end
    archive_path(path, records_with_content)
  end

  def archive_path(path, records_with_content)
    File.open(path, "wb") do |file|
{'fraction_non_alphanumeric': 0.0823937554206418, 'fraction_numerical': 0.0013009540329575022, 'mean_word_length': 3.0859167404782992, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5313463', 'n_tokens_mistral': 1432, 'n_tokens_neox': 1388, 'n_words': 421}