<commit_before>module Bplmodels
  module DescMetadataExport
    extend ActiveSupport::Concern
    included do

      def identifiers_for_export_hash
        ids = []
        descMetadata.identifier.each_with_index.map do |id, index|
          id_type = descMetadata.identifier(index).type_at[0]
          next if id_type == 'uri' && self.class != Bplmodels::OAIObject
          id_hash = { label: id, type: id_type }
          id_hash[:invalid] = true if descMetadata.identifier(index).invalid[0] == 'yes'
          ids << id_hash
        end
        if self.class == Bplmodels::OAIObject
          descMetadata.item_location.url.each_with_index do |url, index|
            url_type = descMetadata.item_location.url(index).access[0]
            ids << { label: url, type: 'uri-preview' } if url_type == 'preview'
          end
        end
        ids
      end

      # TODO: raise an error if title[:primary] is blank?
      def titles_for_export_hash
        titles = { other: [] }
        descMetadata.mods(0).title.each_with_index do |title, index|
          primary = false
          title_prefix = self.descMetadata.mods(0).title_info(index).nonSort[0].presence || ''
          supplied = descMetadata.mods(0).title_info(index).supplied[0].presence
          title_id = descMetadata.mods(0).title_info(index).valueURI[0].presence
          display_label = descMetadata.mods(0).title_info(index).display_label[0].presence
          usage = descMetadata.mods(0).title_info(index).usage[0].presence
          primary = true if display_label == 'primary_display' || (display_label && usage == 'primary')
          title_hash = {
            label: title_prefix + title,
            subtitle: descMetadata.mods(0).title_info(index).subtitle[0].presence,
            type: if primary
                    descMetadata.mods(0).title_info(index).type[0]
                  else
                    descMetadata.mods(0).title_info(index).type[0].presence || 'alternative'
                  end,
            display: primary ? 'primary' : display_label,
            usage: usage,
            supplied: (supplied == 'yes' ? true : nil),
            language: descMetadata.mods(0).title_info(index).language[0].presence,
            authority_code: normalize_authority(descMetadata.mods(0).title_info(index).authority[0].presence),
            id_from_auth: (title_id ? title_id.match(/[A-Za-z0-9]*\z/).to_s : nil),
            part_number: descMetadata.mods(0).title_info(index).part_number[0].presence,
            part_name: descMetadata.mods(0).title_info(index).part_name[0].presence
          }
          title_hash.compact!
          if primary
            titles[:primary] = title_hash
          else
            titles[:other] << title_hash
          end
        end
        titles.reject { |_k, v| v.blank? }
      end

      def names_for_export_hash
        names = []
        descMetadata.mods(0).name.each_with_index do |_name, index|
          nametype = descMetadata.mods(0).name(index).type[0].presence
          name_id = descMetadata.mods(0).name(index).valueURI[0].presence
          name_hash = {
            name_type: nametype,
            authority_code: normalize_authority(descMetadata.mods(0).name(index).authority[0].presence),
            id_from_auth: (name_id ? name_id.match(/[A-Za-z0-9]*\z/).to_s : nil)
          }
          # beware of empty <mods:namePart/>
          name_parts = descMetadata.mods(0).name(index).namePart.reject { |np| np.blank? }
          name_hash[:label] = if nametype == 'corporate'
                                name_parts.join(". ").gsub(/\.\./, '.')
                              else
                                name_parts.join(", ")
                              end
          name_hash[:label].gsub(/[,]{2,}/, ',')
          roles = []
          descMetadata.mods(0).name(index).role.each_with_index do |_role, role_index|
            role_id = descMetadata.mods(0).name(index).role.text.valueURI[role_index]
            role_hash = {
              label: descMetadata.mods(0).name(index).role.text[role_index],
              authority_code: 'marcrelator',
              id_from_auth: (role_id ? role_id.match(/[A-Za-z0-9]*\z/).to_s : nil)
            }
            roles << role_hash.compact
          end
          roles.each do |role_hash|
            names << { name: name_hash.compact, role: role_hash }
          end
        end
        names.uniq
      end<commit_msg>de-dupe identifiers in export hash<commit_after>module Bplmodels
  module DescMetadataExport
    extend ActiveSupport::Concern
    included do

      def identifiers_for_export_hash
        ids = []
        descMetadata.identifier.each_with_index.map do |id, index|
          id_type = descMetadata.identifier(index).type_at[0]
          next if id_type == 'uri' && self.class != Bplmodels::OAIObject
          id_hash = { label: id, type: id_type }
          id_hash[:invalid] = true if descMetadata.identifier(index).invalid[0] == 'yes'
          ids << id_hash
        end
        if self.class == Bplmodels::OAIObject
          descMetadata.item_location.url.each_with_index do |url, index|
            url_type = descMetadata.item_location.url(index).access[0]
            ids << { label: url, type: 'uri-preview' } if url_type == 'preview'
          end
        end
        ids.uniq
      end

      # TODO: raise an error if title[:primary] is blank?
      def titles_for_export_hash
        titles = { other: [] }
        descMetadata.mods(0).title.each_with_index do |title, index|
          primary = false
          title_prefix = self.descMetadata.mods(0).title_info(index).nonSort[0].presence || ''
          supplied = descMetadata.mods(0).title_info(index).supplied[0].presence
          title_id = descMetadata.mods(0).title_info(index).valueURI[0].presence
          display_label = descMetadata.mods(0).title_info(index).display_label[0].presence
          usage = descMetadata.mods(0).title_info(index).usage[0].presence
          primary = true if display_label == 'primary_display' || (display_label && usage == 'primary')
          title_hash = {
            label: title_prefix + title,
            subtitle: descMetadata.mods(0).title_info(index).subtitle[0].presence,
            type: if primary
                    descMetadata.mods(0).title_info(index).type[0]
                  else
                    descMetadata.mods(0).title_info(index).type[0].presence || 'alternative'
                  end,
            display: primary ? 'primary' : display_label,
            usage: usage,
            supplied: (supplied == 'yes' ? true : nil),
            language: descMetadata.mods(0).title_info(index).language[0].presence,
            authority_code: normalize_authority(descMetadata.mods(0).title_info(index).authority[0].presence),
            id_from_auth: (title_id ? title_id.match(/[A-Za-z0-9]*\z/).to_s : nil),
            part_number: descMetadata.mods(0).title_info(index).part_number[0].presence,
            part_name: descMetadata.mods(0).title_info(index).part_name[0].presence
          }
          title_hash.compact!
          if primary
            titles[:primary] = title_hash
          else
            titles[:other] << title_hash
          end
        end
        titles.reject { |_k, v| v.blank? }
      end

      def names_for_export_hash
        names = []
        descMetadata.mods(0).name.each_with_index do |_name, index|
          nametype = descMetadata.mods(0).name(index).type[0].presence
          name_id = descMetadata.mods(0).name(index).valueURI[0].presence
          name_hash = {
            name_type: nametype,
            authority_code: normalize_authority(descMetadata.mods(0).name(index).authority[0].presence),
            id_from_auth: (name_id ? name_id.match(/[A-Za-z0-9]*\z/).to_s : nil)
          }
          # beware of empty <mods:namePart/>
          name_parts = descMetadata.mods(0).name(index).namePart.reject { |np| np.blank? }
          name_hash[:label] = if nametype == 'corporate'
                                name_parts.join(". ").gsub(/\.\./, '.')
                              else
                                name_parts.join(", ")
                              end
          name_hash[:label].gsub(/[,]{2,}/, ',')
          roles = []
          descMetadata.mods(0).name(index).role.each_with_index do |_role, role_index|
            role_id = descMetadata.mods(0).name(index).role.text.valueURI[role_index]
            role_hash = {
              label: descMetadata.mods(0).name(index).role.text[role_index],
              authority_code: 'marcrelator',
              id_from_auth: (role_id ? role_id.match(/[A-Za-z0-9]*\z/).to_s : nil)
            }
            roles << role_hash.compact
          end
          roles.each do |role_hash|
            names << { name: name_hash.compact, role: role_hash }
          end
        end
        names.uniq

        # check for quasi-duplicates (one name with/out id_from_auth etc)
        # some starter code below, but not going to implement this now,
        # need to see how widespread of an issue this is
        # names.uniq!
        # all_namerole_labels = names.map { |v| [v[:name][:label], v[:role][:label]] }
        # if all_namerole_labels.count != all_namerole_labels.uniq.count
        #   0.upto names.length do |_nindex|
        #    removal_candidates = []
        #    keeper_candidates = []
        #    keeper = nil
        #  end
        # end
      end
{'fraction_non_alphanumeric': 0.12250290666948525, 'fraction_numerical': 0.009829827713772329, 'mean_word_length': 2.0463618802318093, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1081871', 'n_tokens_mistral': 2913, 'n_tokens_neox': 2888, 'n_words': 479}