<commit_before>class Ndc < ApplicationRecord
  include PgSearch

  belongs_to :location
  has_many :ndc_targets, class_name: 'NdcSdg::NdcTarget', dependent: :destroy

  validates :document_type, inclusion: {in: %w(ndc indc)}

  attr_accessor :linkages

  PG_SEARCH_HIGHLIGHT_START = '<span class="highlight">'.freeze
  PG_SEARCH_HIGHLIGHT_END = '</span>'.freeze
  PG_SEARCH_HIGHLIGHT_FRAGMENT_DELIMITER = '[[FRAGMENT DELIMITER]]'.freeze
  PG_SEARCH_TSEARCH_DICTIONARY = 'simple'.freeze
  PG_SEARCH_TSEARCH_OPTIONS = {
    dictionary: PG_SEARCH_TSEARCH_DICTIONARY,
    tsvector_column: :full_text_tsv,
    normalization: 4,
    highlight: {
      MaxWords: 30,
      MinWords: 10,
      ShortWord: 3,
      HighlightAll: false,
      MaxFragments: 100,
      StartSel: PG_SEARCH_HIGHLIGHT_START,
      StopSel: PG_SEARCH_HIGHLIGHT_END,
      FragmentDelimiter: PG_SEARCH_HIGHLIGHT_FRAGMENT_DELIMITER
    }
  }.freeze

  # this scope to be used to return results with matches highlighted in
  # fragments
  pg_search_scope :with_highlights_in_fragments, against: :full_text, using: {
    tsearch: PG_SEARCH_TSEARCH_OPTIONS
  }

  # this scope to be used to return results with matches highlighted in
  # full document
  pg_search_scope :with_highlights_in_full, against: :full_text, using: {
    tsearch: PG_SEARCH_TSEARCH_OPTIONS.deep_merge(
      highlight: {
        HighlightAll: true,
        MaxFragments: 0
      }
    )
  }

  def self.refresh_full_text_tsv
    sql = <<~EOT
      full_text_tsv = to_tsvector(
        '#{PG_SEARCH_TSEARCH_DICTIONARY}',
        REGEXP_REPLACE(
          COALESCE(full_text, ''),
          '<sub>(\d+?)</sub>',
          E'\\1',
          'g'
        )
      )
    EOT
    update_all(sql)
  end

  def self.linkages(params)
    query_params = {}

    filters = {
      target: [:ndc_sdg_targets, :number, params[:target]],
      sector: [:ndc_sdg_ndc_target_sectors, :sector_id, params[:sector]],<commit_msg>fixes backslashes<commit_after>class Ndc < ApplicationRecord
  include PgSearch

  belongs_to :location
  has_many :ndc_targets, class_name: 'NdcSdg::NdcTarget', dependent: :destroy

  validates :document_type, inclusion: {in: %w(ndc indc)}

  attr_accessor :linkages

  PG_SEARCH_HIGHLIGHT_START = '<span class="highlight">'.freeze
  PG_SEARCH_HIGHLIGHT_END = '</span>'.freeze
  PG_SEARCH_HIGHLIGHT_FRAGMENT_DELIMITER = '[[FRAGMENT DELIMITER]]'.freeze
  PG_SEARCH_TSEARCH_DICTIONARY = 'simple'.freeze
  PG_SEARCH_TSEARCH_OPTIONS = {
    dictionary: PG_SEARCH_TSEARCH_DICTIONARY,
    tsvector_column: :full_text_tsv,
    normalization: 4,
    highlight: {
      MaxWords: 30,
      MinWords: 10,
      ShortWord: 3,
      HighlightAll: false,
      MaxFragments: 100,
      StartSel: PG_SEARCH_HIGHLIGHT_START,
      StopSel: PG_SEARCH_HIGHLIGHT_END,
      FragmentDelimiter: PG_SEARCH_HIGHLIGHT_FRAGMENT_DELIMITER
    }
  }.freeze

  # this scope to be used to return results with matches highlighted in
  # fragments
  pg_search_scope :with_highlights_in_fragments, against: :full_text, using: {
    tsearch: PG_SEARCH_TSEARCH_OPTIONS
  }

  # this scope to be used to return results with matches highlighted in
  # full document
  pg_search_scope :with_highlights_in_full, against: :full_text, using: {
    tsearch: PG_SEARCH_TSEARCH_OPTIONS.deep_merge(
      highlight: {
        HighlightAll: true,
        MaxFragments: 0
      }
    )
  }

  def self.refresh_full_text_tsv
    sql = <<~EOT
      full_text_tsv = to_tsvector(
        '#{PG_SEARCH_TSEARCH_DICTIONARY}',
        REGEXP_REPLACE(
          COALESCE(full_text, ''),
          '<sub>(\\d+?)</sub>',
          E'\\\\1',
          'g'
        )
      )
    EOT
    update_all(sql)
  end

  def self.linkages(params)
    query_params = {}

    filters = {
      target: [:ndc_sdg_targets, :number, params[:target]],
      sector: [:ndc_sdg_ndc_target_sectors, :sector_id, params[:sector]],
{'fraction_non_alphanumeric': 0.0966914593485509, 'fraction_numerical': 0.0056424724288279045, 'mean_word_length': 3.4167610419026047, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7414842', 'n_tokens_mistral': 1469, 'n_tokens_neox': 1411, 'n_words': 258}