<commit_before>#
# Instance methods for all model classes within the application
#

module Model

  module Core

    def self.included(base)
      base.send :include, Mongoid::Document

      # useful extras, see: http://mongoid.org/en/mongoid/docs/extras.html
      base.send :include, Mongoid::Paranoia # soft deletes
      base.send :index, { deleted_at: 1 }#, { sparse: true }

      base.send :include, Mongoid::Timestamps
      base.send :index, { created_at: 1 }#, { sparse: true }
      base.send :index, { updated_at: 1 }#, { sparse: true }

      base.send :include, Mongoid::Tree
      #base.send :include, Mongoid::Tree::Ordering
      base.send :include, Mongoid::History::Trackable

      # Pagination
      base.send :include, Kaminari::MongoidExtension::Criteria
      base.send :include, Kaminari::MongoidExtension::Document

      # ElasticSearch integration
      # don't index group since they are only a structural construct
      unless 'Group' == base.name
        base.send :include, Tire::Model::Search
        base.send :include, Tire::Model::Callbacks2 # local patched version
        base.send :index_name, base.database_name
      end

      # Generate MD5 fingerprint for this document
      base.send :field, :md5, type: Moped::BSON::Binary
      base.send :index, { md5: 1 }#, { unique: true }
      base.send :set_callback, :save, :before, :generate_md5

      # Make :headings a readable class variable
      base.send :class_eval, %(class << self; attr_reader :headings end)

      # Create rdf_types field and accessor
      base.send :class_eval, %(class << self; attr_reader :rdf_types end)
      base.send :field, :rdf_types

      # Include default embedded vocabularies
      base.send :embeds_one, :dbpedia,  class_name: 'DBpedia',  cascade_callbacks: true, autobuild: true
      base.send :embeds_one, :rdfs,     class_name: 'RDFS',     cascade_callbacks: true, autobuild: true

      # add useful class methods
      # NB: This has to be at the end to monkey-patch Tire, Kaminari, etc.
      base.extend ClassMethods
    end

    def to_normalized_hash(opts={})
      # get a hash that we can modify
      opts[:localize] ? hash = self.to_hash : hash = Hash[self.as_document]

      self.class.normalize(Marshal.load(Marshal.dump(hash)), opts)
    end

    def generate_md5
      hash = self.to_normalized_hash({:ids => :omit})

      self.md5 = Moped::BSON::Binary.new(:md5, Digest::MD5.digest(hash.to_string_recursive.normalize))
    end

    # Retrieve a hash of field names and embedded vocab objects
    def vocabs
      vocabs = {}

      self.class.vocabs.keys.each do |vocab|
        vocabs[vocab] = self.send(vocab) unless self[vocab].nil?
      end

      vocabs
    end

    # Assign model vocab objects by a hash of field names
    def vocabs=(hash)
      update_attributes(hash)
    end

    def locales
      items = self.to_normalized_hash.values.map do |vocab|
        vocab.map do |field, locales|
          next unless locales.is_a? Hash
          locales.keys
        end
      end

      items.flatten.compact.uniq
    end

    def heading(opts={})
      self.class.headings.each do |heading|
        vocab = heading.keys.first
        field = heading.values.first

        unless send(vocab).nil?
          # NB: default is localized
          if opts[:delocalize]
            value = send(vocab)[field]
            value = value.symbolize_keys unless value.nil?
          else
            value = send(vocab).send(field)
          end
          return value unless value.nil?
        end
      end

      # FIXME: return 'untitled' for no heading
      opts[:delocalize] ? {I18n.locale => [I18n.t('model.untitled')]} : [I18n.t('model.untitled')]
    end

    def heading_ancestors(opts={})
      test = ancestors_and_self.map do |node|
        node.heading(opts)
      end

      # NB: default is localized
      if opts[:delocalize]
        test.reduce({}) do |h,pairs|
          pairs.each do |k,v|
            (h[k] ||= []) << v.first
          end
          h
        end
      else
        test.map(&:first)
      end

    end

    def amatch(model, opts={})
      options = {:hamming_similar => true,
                 :jaro_similar => true,
                 :jarowinkler_similar => true,
                 :levenshtein_similar => true,
                 :longest_subsequence_similar => true,
                 :longest_substring_similar => true,
                 :pair_distance_similar => true}

      # if we have selected specific comparisons, use those
      options = opts if opts.is_a? Hash and ! opts.empty?

      p1 = self.to_normalized_hash(options.slice(:ids)).to_string_recursive.normalize
      p2 = model.to_normalized_hash(options.slice(:ids)).to_string_recursive.normalize

      # calculate amatch score for each algorithm
      options.delete :ids
      options.each do |sim, bool|
        options[sim] = p1.send(sim, p2) if bool
      end

      options
    end

    # Search the index and return a Tire::Collection of documents that have a similarity score
    def similar(opts={})
      hash = self.to_normalized_hash
      vocabs = self.vocabs
      id = self.id

      results = self.class.search do
        query do
          boolean do
            # do not include self
            must_not { term :_id, id.to_s }

            # do a structure-free search
            should { match '_all', hash.to_string_recursive.normalize(:space_char => ' ')}

            # NB: use this as a template for recursing in normalized documents?
            hash.each do |name, vocab|
              vocab.each do |field, locales|
                locales.each do |locale, values|
                  # FIXME: temporary workaround for non-localized (dynamic) fields
                  next if values.nil?

                  values.each do |value|
                    should do
                      match "#{name}.#{field}.#{locale}", \
                            value.to_s.normalize({:space_char => ' '}).truncate(100, :separator => ' ')
                    end
                  end
                end
              end
            end
          end
        end
        min_score 1
      end

      # find the maximum score
      maximum = results.max_by {|result| result._score}._score unless results.empty?
<commit_msg>Fix to prevent "too many clauses"error in similarity query<commit_after>#
# Instance methods for all model classes within the application
#

module Model

  module Core

    def self.included(base)
      base.send :include, Mongoid::Document

      # useful extras, see: http://mongoid.org/en/mongoid/docs/extras.html
      base.send :include, Mongoid::Paranoia # soft deletes
      base.send :index, { deleted_at: 1 }#, { sparse: true }

      base.send :include, Mongoid::Timestamps
      base.send :index, { created_at: 1 }#, { sparse: true }
      base.send :index, { updated_at: 1 }#, { sparse: true }

      base.send :include, Mongoid::Tree
      #base.send :include, Mongoid::Tree::Ordering
      base.send :include, Mongoid::History::Trackable

      # Pagination
      base.send :include, Kaminari::MongoidExtension::Criteria
      base.send :include, Kaminari::MongoidExtension::Document

      # ElasticSearch integration
      # don't index group since they are only a structural construct
      unless 'Group' == base.name
        base.send :include, Tire::Model::Search
        base.send :include, Tire::Model::Callbacks2 # local patched version
        base.send :index_name, base.database_name
      end

      # Generate MD5 fingerprint for this document
      base.send :field, :md5, type: Moped::BSON::Binary
      base.send :index, { md5: 1 }#, { unique: true }
      base.send :set_callback, :save, :before, :generate_md5

      # Make :headings a readable class variable
      base.send :class_eval, %(class << self; attr_reader :headings end)

      # Create rdf_types field and accessor
      base.send :class_eval, %(class << self; attr_reader :rdf_types end)
      base.send :field, :rdf_types

      # Include default embedded vocabularies
      base.send :embeds_one, :dbpedia,  class_name: 'DBpedia',  cascade_callbacks: true, autobuild: true
      base.send :embeds_one, :rdfs,     class_name: 'RDFS',     cascade_callbacks: true, autobuild: true

      # add useful class methods
      # NB: This has to be at the end to monkey-patch Tire, Kaminari, etc.
      base.extend ClassMethods
    end

    def to_normalized_hash(opts={})
      # get a hash that we can modify
      opts[:localize] ? hash = self.to_hash : hash = Hash[self.as_document]

      self.class.normalize(Marshal.load(Marshal.dump(hash)), opts)
    end

    def generate_md5
      hash = self.to_normalized_hash({:ids => :omit})

      self.md5 = Moped::BSON::Binary.new(:md5, Digest::MD5.digest(hash.to_string_recursive.normalize))
    end

    # Retrieve a hash of field names and embedded vocab objects
    def vocabs
      vocabs = {}

      self.class.vocabs.keys.each do |vocab|
        vocabs[vocab] = self.send(vocab) unless self[vocab].nil?
      end

      vocabs
    end

    # Assign model vocab objects by a hash of field names
    def vocabs=(hash)
      update_attributes(hash)
    end

    def locales
      items = self.to_normalized_hash.values.map do |vocab|
        vocab.map do |field, locales|
          next unless locales.is_a? Hash
          locales.keys
        end
      end

      items.flatten.compact.uniq
    end

    def heading(opts={})
      self.class.headings.each do |heading|
        vocab = heading.keys.first
        field = heading.values.first

        unless send(vocab).nil?
          # NB: default is localized
          if opts[:delocalize]
            value = send(vocab)[field]
            value = value.symbolize_keys unless value.nil?
          else
            value = send(vocab).send(field)
          end
          return value unless value.nil?
        end
      end

      # FIXME: return 'untitled' for no heading
      opts[:delocalize] ? {I18n.locale => [I18n.t('model.untitled')]} : [I18n.t('model.untitled')]
    end

    def heading_ancestors(opts={})
      test = ancestors_and_self.map do |node|
        node.heading(opts)
      end

      # NB: default is localized
      if opts[:delocalize]
        test.reduce({}) do |h,pairs|
          pairs.each do |k,v|
            (h[k] ||= []) << v.first
          end
          h
        end
      else
        test.map(&:first)
      end

    end

    def amatch(model, opts={})
      options = {:hamming_similar => true,
                 :jaro_similar => true,
                 :jarowinkler_similar => true,
                 :levenshtein_similar => true,
                 :longest_subsequence_similar => true,
                 :longest_substring_similar => true,
                 :pair_distance_similar => true}

      # if we have selected specific comparisons, use those
      options = opts if opts.is_a? Hash and ! opts.empty?

      p1 = self.to_normalized_hash(options.slice(:ids)).to_string_recursive.normalize
      p2 = model.to_normalized_hash(options.slice(:ids)).to_string_recursive.normalize

      # calculate amatch score for each algorithm
      options.delete :ids
      options.each do |sim, bool|
        options[sim] = p1.send(sim, p2) if bool
      end

      options
    end

    # Search the index and return a Tire::Collection of documents that have a similarity score
    def similar(opts={})
      hash = self.to_normalized_hash
      vocabs = self.vocabs
      id = self.id

      results = self.class.search do
        query do
          boolean do
            # do not include self
            must_not { term :_id, id.to_s }

            # do a structure-free search
            should { match '_all', hash.to_string_recursive.normalize(:space_char => ' ').truncate(4096, :separator => ' ', :omission => '')}

            # NB: use this as a template for recursing in normalized documents?
            hash.each do |name, vocab|
              vocab.each do |field, locales|
                locales.each do |locale, values|
                  # FIXME: temporary workaround for non-localized (dynamic) fields
                  next if values.nil?

                  values.each do |value|
                    should do
                      match "#{name}.#{field}.#{locale}", \
                            value.to_s.normalize({:space_char => ' '}).truncate(4096, :separator => ' ', :omission => '')
                    end
                  end
                end
              end
            end
          end
        end
        min_score 1
      end

      # find the maximum score
      maximum = results.max_by {|result| result._score}._score unless results.empty?

{'fraction_non_alphanumeric': 0.1003462930898788, 'fraction_numerical': 0.004643475523374784, 'mean_word_length': 2.1282619399310683, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1021695', 'n_tokens_mistral': 3939, 'n_tokens_neox': 3752, 'n_words': 1022}