<commit_before>class QueryBuilder

  def self.detect_duplicates_of(org_flat)
    {
        query: v2(org_flat),
        size: 5
    }
  end

  private

  def self.v2(org)
    {
        bool: {
            should: [
                #{
                #    fuzzy_like_this: {
                #        like_text: org['id'] || '',
                #        fields: ['id'],
                #        max_query_terms: 1,
                #        min_similarity: 0.9
                #    }
                #},
                {
                    fuzzy_like_this: {
                        like_text: org['name'] || '',
                        fields: %w(name),
                        min_similarity: 0.5,
                        boost: 5.0
                    }
                },
                {
                    fuzzy_like_this: {
                        like_text: org['gov_id1'] || '',
                        fields: %w(gov_id1),
                        prefix_length: 4,
                        min_similarity: 0.9,
                        boost: 10.0
                    }
                },
                {
                    fuzzy_like_this: {
                        like_text: org['city'] || '',
                        fields: %w(city),
                        min_similarity: 0.6,
                        boost: 4.0
                    }
                },
                {
                    fuzzy_like_this: {
                        like_text: org['state'] || '',
                        fields: %w(state),
                        max_query_terms: 1,
                        min_similarity: 0.5,
                        boost: 4.0
                    }
                },
                #{
                #    fuzzy_like_this: {
                #        like_text: org['country'] || '',
                #        fields: ['country'],
                #        max_query_terms: 3,
                #        min_similarity: 0.8,
                #        boost: 1
                #    }
                #},
            ],
            must: [],
            minimum_should_match: 1
        }
    }
  end

end<commit_msg>query: wip<commit_after>class QueryBuilder

  def self.detect_duplicates_of(org_flat)
    {
        query: v2(org_flat),
        size: 5
    }
  end

  private

  def self.v2(org)
    {
        bool: {
            should: [
                {
                    fuzzy_like_this: {
                        like_text: org['name'] || '',
                        fields: %w(name),
                        min_similarity: 0.5,
                        boost: 5.0
                    }
                },
                {
                    term: {
                        gov_id1: {
                            term: org['gov_id1'] || '',
                            boost: 10.0
                        }
                    }
                },
                {
                    fuzzy_like_this: {
                        like_text: org['city'] || '',
                        fields: %w(city),
                        min_similarity: 0.7,
                        boost: 4.0
                    }
                },
                {
                    match_phrase: {
                        state: {
                            query: org['state'] || '',
                            boost: 4.0
                        }
                    }
                },
            ],
            must: [],
            minimum_should_match: 2
        }
    }
  end

end
{'fraction_non_alphanumeric': 0.09551043751787246, 'fraction_numerical': 0.014583929082070346, 'mean_word_length': 0.5770964833183048, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4354620', 'n_tokens_mistral': 963, 'n_tokens_neox': 832, 'n_words': 128}