<commit_before># encoding: utf-8

require 'libis/ingester'
require 'libis/tools/extend/hash'

module Libis
  module Ingester

    class IntellectualEntity < Libis::Ingester::Item
      include Libis::Workflow::Mongoid::Base

      field :ingest_type, type: String, default: 'METS'
      field :pid, type: String

      index({pid: 1}, {sparse: true, name: 'by_pid'})

      belongs_to :retention_period, class_name: Libis::Ingester::RetentionPeriod.to_s, inverse_of: nil

      def representations
        self.items.select { |item| item.is_a? ::Libis::Ingester::Representation }
      end

      def representation(name_or_id)
        representations.each do |representation|
          return representation if name_or_id == representation.id or name_or_id == representation.name
        end
        nil
      end

      def originals
        self.items.reject { |item| item.is_a? ::Libis::Ingester::Representation }
      end

      # noinspection RubyResolve
      def to_hash
        result = super
        result[:retention_period_id] = self.retention_period.rp_id if self.retention_period
        result.cleanup
      end

    end

  end
end<commit_msg>Intellectual Entity: improve Mongo queries for speed.<commit_after># encoding: utf-8

require 'libis/ingester'
require 'libis/tools/extend/hash'

module Libis
  module Ingester

    class IntellectualEntity < Libis::Ingester::Item
      include Libis::Workflow::Mongoid::Base

      field :ingest_type, type: String, default: 'METS'
      field :pid, type: String

      index({pid: 1}, {sparse: true, name: 'by_pid'})

      belongs_to :retention_period, class_name: Libis::Ingester::RetentionPeriod.to_s, inverse_of: nil

      def representations
        self.items.where(_type: Libis::Ingester::Representation.to_s)
      end

      def representation(name_or_id)
        self.representations.where(id: name_or_id).first || self.representations.where(name: name_or_id).first
      end

      def originals
        self.items.ne(_type: Libis::Ingester::Representation.to_s)
      end

      # noinspection RubyResolve
      def to_hash
        result = super
        result[:retention_period_id] = self.retention_period.rp_id if self.retention_period
        result.cleanup
      end

    end

  end
end
{'fraction_non_alphanumeric': 0.09403973509933775, 'fraction_numerical': 0.0017660044150110375, 'mean_word_length': 3.097649186256781, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4999588', 'n_tokens_mistral': 725, 'n_tokens_neox': 690, 'n_words': 158}