<commit_before># frozen_string_literal: true

module GobiertoCommon
  module Searchable
    extend ActiveSupport::Concern

    included do
      include AlgoliaSearch
      include ActionView::Helpers::SanitizeHelper

      def self.search_index_name
        "#{APP_CONFIG["site"]["name"]}_#{Rails.env}_#{name}"
      end

      def self.algoliasearch_gobierto(&block)
        algoliasearch(enqueue: :trigger_reindex_job, disable_indexing: Rails.env.test?, index_name: search_index_name, if: :active?, sanitize: true, &block)
      end

      def self.trigger_reindex_job(record, remove)
        return if record.nil? || (record.respond_to?(:site) && record.site.algolia_search_disabled?)

        GobiertoCommon::AlgoliaReindexJob.perform_later(record.class.name, record.id, remove)
      end
    end

    def class_name
      self.class.name
    end

    def searchable_attribute(value)
      return "" unless value.present?

      strip_tags(value.tr("\n\r", " ").gsub(/\s+/, " "))[0..9300]
    end

    def searchable_translated_attribute(translations_hash)
      return "" if translations_hash.nil?

      attribute_summary = translations_hash.values.join(" ").tr("\n\r", " ").gsub(/\s+/, " ")
      attribute_summary = strip_tags(attribute_summary)
      attribute_summary[0..9300]
    end
  end
end
<commit_msg>Include pg_search support in searchable concern<commit_after># frozen_string_literal: true

module GobiertoCommon
  module Searchable
    extend ActiveSupport::Concern

    included do
      include AlgoliaSearch
      include ActionView::Helpers::SanitizeHelper
      include PgSearch::Model

      def self.search_index_name
        "#{APP_CONFIG["site"]["name"]}_#{Rails.env}_#{name}"
      end

      def self.algoliasearch_gobierto(&block)
        algoliasearch(enqueue: :trigger_reindex_job, disable_indexing: Rails.env.test?, index_name: search_index_name, if: :active?, sanitize: true, &block)
      end

      def self.trigger_reindex_job(record, remove)
        return if record.nil? || (record.respond_to?(:site) && record.site.algolia_search_disabled?)

        GobiertoCommon::AlgoliaReindexJob.perform_later(record.class.name, record.id, remove)
      end
    end

    def class_name
      self.class.name
    end

    def searchable_attribute(value)
      return "" unless value.present?

      strip_tags(value.tr("\n\r", " ").gsub(/\s+/, " "))[0..9300]
    end

    def searchable_translated_attribute(translations_hash)
      return "" if translations_hash.nil?

      attribute_summary = translations_hash.values.join(" ").tr("\n\r", " ").gsub(/\s+/, " ")
      attribute_summary = strip_tags(attribute_summary)
      attribute_summary[0..9300]
    end

    # pg_search

    def truncated_translations(attribute, limit: 1000, strip_tags: true)
      return {} unless (translations_hash = send("#{attribute}_translations")).present?

      translations_hash.transform_values { |value| (strip_tags ? strip_tags(value) : value)&.truncate(limit) }
    end
  end
end

{'fraction_non_alphanumeric': 0.11870623541180393, 'fraction_numerical': 0.008002667555851951, 'mean_word_length': 3.5454545454545454, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7480446', 'n_tokens_mistral': 976, 'n_tokens_neox': 940, 'n_words': 172}