<commit_before>module RefugeeIndex
  extend ActiveSupport::Concern

  included do
    include Elasticsearch::Model
    # include Elasticsearch::Model::Callbacks

    settings Rails.application.config.elasticsearch

    # Override model name
    index_name "refugees_#{Rails.env}"
    document_type 'refugee'

    after_commit do
      __elasticsearch__.index_document
    end

    after_destroy do
      delete_document
    end

    mappings dynamic: 'false' do
      indexes :name, analyzer: 'simple'
      indexes :name_phrase, analyzer: 'simple'
      indexes :name_search, analyzer: 'name_index', search_analyzer: 'name_search'
      indexes :ssn, analyzer: 'nmbr', search_analyzer: 'standard'
      indexes :ssns, analyzer: 'nmbr_letters', search_analyzer: 'standard'
      indexes :dossier_number, analyzer: 'nmbr', search_analyzer: 'standard'
      indexes :dossier_numbers, analyzer: 'nmbr_letters', search_analyzer: 'standard'
    end

    def delete_document
      # ES document might already be deleted, so we do not log errors unless debug
      begin
        __elasticsearch__.delete_document
      rescue Exception => e
        logger.debug { "Document could not be deleted: #{e}" }
      end
    end
  end

  def as_indexed_json(options={})
    {
      id: id,
      name: name,
      name_phrase: name,
      name_search: name,
      ssn: ssn,
      ssns: ssns.map(&:full_ssn).join(' '),
      dossier_number: dossier_number,
      dossier_numbers: dossier_numbers.map(&:name).join(' ')
    }.as_json
  end
end<commit_msg>Fixed bug with deleting from ES index<commit_after>module RefugeeIndex
  extend ActiveSupport::Concern

  included do
    include Elasticsearch::Model
    # include Elasticsearch::Model::Callbacks

    settings Rails.application.config.elasticsearch

    # Override model name
    index_name "refugees_#{Rails.env}"
    document_type 'refugee'

    after_commit -> { __elasticsearch__.index_document  },  on: [:create, :update]
    after_commit -> { __elasticsearch__.delete_document },  on: :destroy

    mappings dynamic: 'false' do
      indexes :name, analyzer: 'simple'
      indexes :name_phrase, analyzer: 'simple'
      indexes :name_search, analyzer: 'name_index', search_analyzer: 'name_search'
      indexes :ssn, analyzer: 'nmbr', search_analyzer: 'standard'
      indexes :ssns, analyzer: 'nmbr_letters', search_analyzer: 'standard'
      indexes :dossier_number, analyzer: 'nmbr', search_analyzer: 'standard'
      indexes :dossier_numbers, analyzer: 'nmbr_letters', search_analyzer: 'standard'
    end
  end

  def as_indexed_json(options={})
    {
      id: id,
      name: name,
      name_phrase: name,
      name_search: name,
      ssn: ssn,
      ssns: ssns.map(&:full_ssn).join(' '),
      dossier_number: dossier_number,
      dossier_numbers: dossier_numbers.map(&:name).join(' ')
    }.as_json
  end
end
{'fraction_non_alphanumeric': 0.09944173063503141, 'fraction_numerical': 0.0, 'mean_word_length': 3.4380804953560373, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3729408', 'n_tokens_mistral': 933, 'n_tokens_neox': 886, 'n_words': 220}