<commit_before>require 'active_support/concern'

module Refinery
  module Elasticsearch
    module Searchable
      extend ActiveSupport::Concern

      def strip_tags(s)
        return nil if s.nil?
        s.gsub(/<[^>]*>/ui, ' ').gsub(/\s+/, ' ')
      end

      def index_document
        if self.respond_to?(:to_index)
          if document = self.to_index
            ::Refinery::Elasticsearch.with_client do |client|
              client.index({
                index: ::Refinery::Elasticsearch.index_name,
                type:  self.class.document_type,
                id:    self.id,
                body:  document
              })
              ::Refinery::Elasticsearch.log(:debug, "Indexed document #{self.id}")
            end
          end
        end
      end

      def update_document
        if self.respond_to?(:to_index)
          if document = self.to_index
            needs_update = !(self.previous_changes.keys & document.keys).empty?
            ::Refinery::Elasticsearch.with_client do |client|
              client.index({
                index: ::Refinery::Elasticsearch.index_name,
                type:  self.class.document_type,
                id:    self.id,
                body:  document
              })
              ::Refinery::Elasticsearch.log(:debug, "Updated document #{self.id}")
            end if needs_update
          end
        end
      end

      def delete_document
        ::Refinery::Elasticsearch.with_client do |client|
          client.delete({
            index: ::Refinery::Elasticsearch.index_name,
            type:  self.class.document_type,
            id:    self.id
          })
          ::Refinery::Elasticsearch.log(:debug, "Deleted document #{self.id}")
        end
      end

      included do
        after_commit :index_document, on: :create
        after_commit :update_document, on: :update
        after_commit :delete_document, on: :destroy
        ::Refinery::Elasticsearch.searchable_classes << self
      end

      module ClassMethods
        def indexable
          # this is meant as a relation which includes all elements. In Rails <4.0
          # the `all` class method does not return a relation but an array.
          where('1=1')
        end

        def define_mapping(&block)
          @mapping = block.call
        end

        def mapping
          @mapping
        end

        def document_type
          @document_type ||= name.underscore.gsub('/', '-')
        end

        def index_all
          indexable.find_in_batches(batch_size:100) do |group|
            bulk_data = group.map do |e|
              [
                { index: { _index: ::Refinery::Elasticsearch.index_name, _type:self.document_type, _id:e.id } }, 
                e.to_index
              ]
            end.flatten

            ::Refinery::Elasticsearch.with_client do |client|
              client.bulk body:bulk_data
            end unless bulk_data.empty?
          end
        end
      end
    end
  end
end
<commit_msg>Fix error preventing partial updates from happen<commit_after>require 'active_support/concern'

module Refinery
  module Elasticsearch
    module Searchable
      extend ActiveSupport::Concern

      def strip_tags(s)
        return nil if s.nil?
        s.gsub(/<[^>]*>/ui, ' ').gsub(/\s+/, ' ')
      end

      def index_document
        if self.respond_to?(:to_index)
          if document = self.to_index
            ::Refinery::Elasticsearch.with_client do |client|
              client.index({
                index: ::Refinery::Elasticsearch.index_name,
                type:  self.class.document_type,
                id:    self.id,
                body:  document
              })
              ::Refinery::Elasticsearch.log(:debug, "Indexed document #{self.id}")
            end
          end
        end
      end

      def update_document
        if self.respond_to?(:to_index)
          if document = self.to_index
            needs_update = !(self.previous_changes.keys.map(&:to_sym) & document.keys.map(&:to_sym)).empty?
            ::Refinery::Elasticsearch.with_client do |client|
              client.index({
                index: ::Refinery::Elasticsearch.index_name,
                type:  self.class.document_type,
                id:    self.id,
                body:  document
              })
              ::Refinery::Elasticsearch.log(:debug, "Updated document #{self.id}")
            end if needs_update
          end
        end
      end

      def delete_document
        ::Refinery::Elasticsearch.with_client do |client|
          client.delete({
            index: ::Refinery::Elasticsearch.index_name,
            type:  self.class.document_type,
            id:    self.id
          })
          ::Refinery::Elasticsearch.log(:debug, "Deleted document #{self.id}")
        end
      end

      included do
        after_commit :index_document, on: :create
        after_commit :update_document, on: :update
        after_commit :delete_document, on: :destroy
        ::Refinery::Elasticsearch.searchable_classes << self
      end

      module ClassMethods
        def indexable
          # this is meant as a relation which includes all elements. In Rails <4.0
          # the `all` class method does not return a relation but an array.
          where('1=1')
        end

        def define_mapping(&block)
          @mapping = block.call
        end

        def mapping
          @mapping
        end

        def document_type
          @document_type ||= name.underscore.gsub('/', '-')
        end

        def index_all
          indexable.find_in_batches(batch_size:100) do |group|
            bulk_data = group.map do |e|
              [
                { index: { _index: ::Refinery::Elasticsearch.index_name, _type:self.document_type, _id:e.id } }, 
                e.to_index
              ]
            end.flatten

            ::Refinery::Elasticsearch.with_client do |client|
              client.bulk body:bulk_data
            end unless bulk_data.empty?
          end
        end
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.09461018625350255, 'fraction_numerical': 0.002307565518378111, 'mean_word_length': 1.6417065737919025, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3135945', 'n_tokens_mistral': 1686, 'n_tokens_neox': 1667, 'n_words': 382}