<commit_before>require 'nokogiri'

class XMLMediaImporter

  attr_accessor :file, :collection, :dir, :collection1, :collection2, :file_name, :max_load_per_collection

  def initialize(options={})
    #PBCore.config[:date_formats] = ['%m/%d/%Y', '%Y-%m-%d']
    if options.has_key?(:collection_id_2)
      self.collection1 = options[:collection_id]
      self.collection2 = options[:collection_id_2]
      self.file_name = options[:file]
      self.max_load_per_collection = Integer(options[:max])
    else
      self.collection = Collection.find(options[:collection_id])
    end
    directory = options[:dir] ||= nil

    if directory == nil
      raise "File missing or 0 length: #{options[:file]}" unless (File.size?(options[:file]).to_i > 0)
      self.file = File.open(options[:file])
      #self.dir = Dir[options[:file]+"*.xml"]
    else
      self.dir = Dir[directory+"/*.xml"]
    end

  end

  def is_audio_file?(url)
    puts "is_audio_file? url:#{url}"
    uri = URI.parse(url)
    ext = (File.extname(uri.path)[1..-1] || "").downcase
    ['mp3', 'wav', 'mp2', 'aac'].include?(ext)
  end

  def import_xml_illinois_collection
    xmlFeed=Nokogiri::XML(file)
    xmlFeed.remove_namespaces!
    descriptions = xmlFeed.xpath("//PBCoreCollection/PBCoreDescriptionDocument")
    descriptions.each do |pbcoreDescription|
      item_for_illinois_doc(pbcoreDescription).save!
    end
  end

  def split_ks_xml_file
    importer_FB = PBCoreImporter.new(collection_id: self.collection1, file: self.file_name)
    importer_B = PBCoreImporter.new(collection_id: self.collection2, file: self.file_name)
    count_FB = 0
    count_F = 0
    pbc_collection = PBCore::V2::Collection.parse(file)
    pbc_collection.description_documents.each do |doc|
      doc.instantiations.each do |pbcInstance|
        url = pbcInstance.detect_element(:identifiers, match_attr: :source, match_value: ['URL', nil])
        if url.match('F-B')
          if count_FB >= max_load_per_collection
            break
          end
          count_FB =count_FB+1
          puts "F-B " + url
          importer_FB.item_for_omeka_doc(doc).save!
        else
          if count_F >= max_load_per_collection
            break
          end
          count_F = count_F+1
          puts "NO F" + url
          importer_B.item_for_omeka_doc(doc).save!
        end
        break
      end
    end
  end

  def import_xml_bbg_feed
    # pbc_collection = PBCore::V2::Collection.parse(file)
    xmlFeed=Nokogiri::XML(file)
    xmlFeed.remove_namespaces!
    items = xmlFeed.xpath("//item")
    items.each do |item|
      item_for_bbg(item).save!
    end
  end


  def import_openvault_directory
    # pbc_collection = PBCore::V2::Collection.parse(file)
    dir.each do |xmlfile|
      file = File.open(xmlfile.to_s)
      xmlFeed=Nokogiri::XML(file)
      #to simplify xpath queries, try to remove namespaces
      #it might not work for files with multiples namespaces
      xmlFeed.remove_namespaces!<commit_msg>adding a 2 seconds sleep to the uploading process<commit_after>require 'nokogiri'

class XMLMediaImporter

  attr_accessor :file, :collection, :dir, :collection1, :collection2, :file_name, :max_load_per_collection

  def initialize(options={})
    #PBCore.config[:date_formats] = ['%m/%d/%Y', '%Y-%m-%d']
    if options.has_key?(:collection_id_2)
      self.collection1 = options[:collection_id]
      self.collection2 = options[:collection_id_2]
      self.file_name = options[:file]
      self.max_load_per_collection = Integer(options[:max])
    else
      self.collection = Collection.find(options[:collection_id])
    end
    directory = options[:dir] ||= nil

    if directory == nil
      raise "File missing or 0 length: #{options[:file]}" unless (File.size?(options[:file]).to_i > 0)
      self.file = File.open(options[:file])
      #self.dir = Dir[options[:file]+"*.xml"]
    else
      self.dir = Dir[directory+"/*.xml"]
    end

  end

  def is_audio_file?(url)
    puts "is_audio_file? url:#{url}"
    uri = URI.parse(url)
    ext = (File.extname(uri.path)[1..-1] || "").downcase
    ['mp3', 'wav', 'mp2', 'aac'].include?(ext)
  end

  def import_xml_illinois_collection
    xmlFeed=Nokogiri::XML(file)
    xmlFeed.remove_namespaces!
    descriptions = xmlFeed.xpath("//PBCoreCollection/PBCoreDescriptionDocument")
    descriptions.each do |pbcoreDescription|
      item_for_illinois_doc(pbcoreDescription).save!
    end
  end

  def split_ks_xml_file
    importer_FB = PBCoreImporter.new(collection_id: self.collection1, file: self.file_name)
    importer_B = PBCoreImporter.new(collection_id: self.collection2, file: self.file_name)
    count_FB = 0
    count_F = 0
    pbc_collection = PBCore::V2::Collection.parse(file)
    pbc_collection.description_documents.each do |doc|
      doc.instantiations.each do |pbcInstance|
        url = pbcInstance.detect_element(:identifiers, match_attr: :source, match_value: ['URL', nil])
        if url.match('F-B')
          if count_FB >= max_load_per_collection
            break
          end
          count_FB =count_FB+1
          puts "F-B " + url
          importer_FB.item_for_omeka_doc(doc).save!
        else
          if count_F >= max_load_per_collection
            break
          end
          count_F = count_F+1
          puts "NO F" + url
          importer_B.item_for_omeka_doc(doc).save!
        end
        break
      end
    end
  end

  def import_xml_bbg_feed
    # pbc_collection = PBCore::V2::Collection.parse(file)
    xmlFeed=Nokogiri::XML(file)
    xmlFeed.remove_namespaces!
    items = xmlFeed.xpath("//item")
    items.each do |item|
      sleep(2)
      item_for_bbg(item).save!
    end
  end


  def import_openvault_directory
    # pbc_collection = PBCore::V2::Collection.parse(file)
    dir.each do |xmlfile|
      file = File.open(xmlfile.to_s)
      xmlFeed=Nokogiri::XML(file)
      #to simplify xpath queries, try to remove namespaces
      #it might not work for files with multiples namespaces
      xmlFeed.remove_namespaces!
{'fraction_non_alphanumeric': 0.11764705882352941, 'fraction_numerical': 0.0073321113147808694, 'mean_word_length': 3.1053351573187413, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 30}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2100763', 'n_tokens_mistral': 2041, 'n_tokens_neox': 2020, 'n_words': 382}