<commit_before>require 'libis-workflow'
require 'libis-ingester'

require 'double_bag_ftps'

module Libis
  module Ingester

    class MasterthesisCollector < Libis::Ingester::Task

      parameter ftp_host: 'ftpsap.cc.kuleuven.be',
                description: 'FTP host where theses are uploaded.'
      parameter ftp_port: 990,
                description: 'FTP port number.'
      parameter ftp_user: 'plias',
                description: 'FTP user account.'
      parameter ftp_password: '<PASSWORD>',
                description: 'FTP password.'
      parameter ftp_subdir: '/masterproef/out',
                description: 'Path where theses are stored.'

      parameter item_types: [Libis::Ingester::Run], frozen: true

      protected

      # @param [Libis::Ingester::Run] item
      def process(item)
        @work_dir = item.work_dir
        loaded = PersistentStorage.where(domain: 'Masterthesis').find_or_create_by(name: 'Loaded')[:data]
        ftp_connect
        dirs = ftp_ls parameter(:ftp_subdir)
        dirs.each do |dir|
          next if is_file?(dir)
          name = File.basename(dir)
          if loaded.has_key?(name)
            warn 'Thesis found that is already ingested: \'%s\' [%s]', name, loaded[name]
            next
          end
          process_dir(dir)
        end
      end

      private

      def process_dir(dir)
        dir_name = File.basename(dir)
        info 'Processing dir %s', dir_name

        work_dir = File.join(@work_dir, dir_name)
        FileUtils.mkpath(work_dir)
        files = ftp_ls(dir)
        files = files.map do |file|
          local_file = File.join(work_dir, File.basename(file))
          ftp_get_file(file, local_file)
          local_file
        end

        xml_file_name = 'e_thesis.xml'
        xml_file = files.find { |file| File.basename(file) == xml_file_name }
        unless xml_file
          error 'XML file missing in %s', dir_name
          #raise Libis::WorkflowError
        end

        xml_doc = Libis::Tools::XmlDocument.open(xml_file)
        proeven = xml_doc.root.search('/proeven/proef')
        warn 'XML file in %s contains multiple theses. Only using first item.', dir_name if proeven.size > 1
        proef = proeven.first

        ie_item = Libis::Ingester::IntellectualEntity.new
        ie_item.name = dir_name
        ie_item.properties['title'] = proef.search('titel1/tekst').map(&:text).map(&:strip).first
        workitem << ie_item
        ie_item.save!
        debug 'Added IE \'%s\'', ie_item.properties['title']

        if ie_item.properties['title'].nil?
          error 'XML entry in does not have a value for titel1/text.', ie_item
          # raise Libis::WorkflowError
          return
        end

        hoofdtekst = proef.search('bestanden/hoofdtekst').map(&:text)

        if hoofdtekst.empty?
          error 'XML file missing a main file entry (bestanden/hoofdtekst).', ie_item
          # raise Libis::WorkflowError
          return
        end

        if hoofdtekst.size > 1
          error 'XML file has multiple a main file entries (bestanden/hoofdtekst).', ie_item
          # raise Libis::WorkflowError
          return
        end

        bijlagen = proef.search('bestanden/bijlage').map(&:text)
        files_from_xml = hoofdtekst + bijlagen

        ok = true

        files_from_xml.each do |fname|
          unless files.any? { |file| File.basename(file) == fname }
            error 'A file \'%s\' listed in the XML is not found on FTP server', ie_item, fname
            ok = false
            next
          end
          file = Libis::Ingester::FileItem.new
          file.filename = File.join(dir, fname)
          ie_item << file
          debug 'Added file \'%s\'.', ie_item, fname
        end

        files.each do |file|
          next if File.basename(file) == xml_file_name
          next if files_from_xml.include?(File.basename(file))
          error 'A file \'%s\' was found on the FTP that was not listed in the XML', ie_item, File.basename(file)
          ok = false
        end

        # raise Libis::WorkflowError unless ok
        return unless ok
<commit_msg>ETD ingest bug fixes<commit_after>require 'libis-workflow'
require 'libis-ingester'

require 'double_bag_ftps'

module Libis
  module Ingester

    class MasterthesisCollector < Libis::Ingester::Task

      parameter ftp_host: 'ftpsap.cc.kuleuven.be',
                description: 'FTP host where theses are uploaded.'
      parameter ftp_port: 990,
                description: 'FTP port number.'
      parameter ftp_user: 'plias',
                description: 'FTP user account.'
      parameter ftp_password: '<PASSWORD>',
                description: 'FTP password.'
      parameter ftp_subdir: '/masterproef/out',
                description: 'Path where theses are stored.'

      parameter item_types: [Libis::Ingester::Run], frozen: true

      protected

      # @param [Libis::Ingester::Run] item
      def process(item)
        @work_dir = item.work_dir
        loaded = PersistentStorage.where(domain: 'Masterthesis').find_or_create_by(name: 'Loaded')[:data]
        ftp_connect
        dirs = ftp_ls parameter(:ftp_subdir)
        dirs.each do |dir|
          next if is_file?(dir)
          name = File.basename(dir)
          if loaded.has_key?(name)
            warn 'Thesis found that is already ingested: \'%s\' [%s]', name, loaded[name]
            next
          end
          process_dir(dir)
        end
      end

      private

      def process_dir(dir)
        dir_name = File.basename(dir)
        info 'Processing dir %s', dir_name

        work_dir = File.join(@work_dir, dir_name)
        FileUtils.mkpath(work_dir)
        files = ftp_ls(dir)
        files = files.map do |file|
          local_file = File.join(work_dir, File.basename(file))
          ftp_get_file(file, local_file)
          local_file
        end

        xml_file_name = 'e_thesis.xml'
        xml_file = files.find { |file| File.basename(file) == xml_file_name }
        unless xml_file
          error 'XML file missing in %s', dir_name
          #raise Libis::WorkflowError
        end

        xml_doc = Libis::Tools::XmlDocument.open(xml_file)
        proeven = xml_doc.root.search('/proeven/proef')
        warn 'XML file in %s contains multiple theses. Only using first item.', dir_name if proeven.size > 1
        proef = proeven.first

        ie_item = Libis::Ingester::IntellectualEntity.new
        ie_item.name = dir_name
        ie_item.properties['title'] = proef.search('titel1/tekst').map(&:text).map(&:strip).first
        workitem << ie_item
        ie_item.save!
        debug 'Added IE \'%s\'', ie_item.properties['title']

        if ie_item.properties['title'].nil?
          error 'XML entry in does not have a value for titel1/text.', ie_item
          # raise Libis::WorkflowError
          return
        end

        hoofdtekst = proef.search('bestanden/hoofdtekst').map(&:text)

        if hoofdtekst.empty?
          error 'XML file missing a main file entry (bestanden/hoofdtekst).', ie_item
          # raise Libis::WorkflowError
          return
        end

        if hoofdtekst.size > 1
          error 'XML file has multiple a main file entries (bestanden/hoofdtekst).', ie_item
          # raise Libis::WorkflowError
          return
        end

        if hoofdtekst.first.blank?
          error 'XML file has an empty main file entry (bestanden/hoofdtekst).', ie_item
          # raise Libis::WorkflowError
          return
        end

        bijlagen = proef.search('bestanden/bijlage').map(&:text)
        files_from_xml = hoofdtekst + bijlagen

        ok = true

        files_from_xml.each do |fname|
          unless files.any? { |file| File.basename(file) == fname }
            error 'A file \'%s\' listed in the XML is not found on FTP server', ie_item, fname
            ok = false
            next
          end
          file = Libis::Ingester::FileItem.new
          file.filename = File.join(dir, fname)
          ie_item << file
          debug 'Added file \'%s\'.', ie_item, fname
        end

        files.each do |file|
          next if File.basename(file) == xml_file_name
          next if files_from_xml.include?(File.basename(file))
          error 'A file \'%s\' was found on the FTP that was not listed in the XML', ie_item, File.basename(file)
          ok = false
        end

        # raise Libis::WorkflowError unless ok
        return unless ok

{'fraction_non_alphanumeric': 0.09936049265750829, 'fraction_numerical': 0.0016579819990525818, 'mean_word_length': 2.1059212945936006, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 24}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '248870', 'n_tokens_mistral': 2620, 'n_tokens_neox': 2550, 'n_words': 664}