<commit_before>require 'nokogiri'
module SearchMethods
  module Sfx4
    include MetadataHelper # for normalize_lccn

    protected
    # Class method for the module that gets called by the umlaut:load_sfx_urls task.
    # Determines whether we should attempt to fetch SFX urls.
    # Will probably be deprecated in the near future.
    def self.fetch_urls?
      sfx4_base.connection_configured?
    end

    # Class method for the module that gets called by the umlaut:load_sfx_urls task.
    # Kind of hacky way of trying to extract target URLs from SFX4.
    # Will probably be deprecated in the near future.
    def self.fetch_urls
      sfx4_base.fetch_urls
    end
    
    # Class method for the module.
    # Returns the SFX4 base class in order to establish a connection.
    def self.sfx4_base
      # Need to do this convoluted Module.const_get so that we find the
      # correct class. Otherwise the module looks locally and can't find it.
      Module.const_get(:Sfx4).const_get(:Local).const_get(:Base)
    end

    # Instance method that returns the SFX4 AzTitle class for this search method.
    # Can be overridden by search methods that want to include this one.
    def az_title_klass
      # Need to do this convoluted Module.const_get so that we find the
      # correct class. Otherwise the module looks locally and can't find it.
      Module.const_get(:Sfx4).const_get(:Local).const_get(:AzTitle)
    end

    # Instance method that returns the SFX4 DB connection for this search method.
    def sfx4_db_connection
      az_title_klass.connection
    end

    # Needs to return ContextObjects
    def find_by_title                  
      connection = sfx4_db_connection
      query_match_clause = case search_type_param
        when "contains"
          terms = title_query_param.split(" ")
          #SFX4 seems to ignore 'the' or 'a' on the front, so we will too. 
          if (["the", "a"].include? terms[0])
            terms = terms.slice(1..-1)
          end
          # Then make each term required, but stemmed. Seems to match SFX4, 
          # and more importantly give us decent results. 
          query = terms.collect do |term|
            "+" + connection.quote_string(term) + "*"
          end.join(" ")
          "MATCH (TS.TITLE_SEARCH) AGAINST ('#{query}' IN BOOLEAN MODE)"
        when "begins"
          # For 'begins', searching against TITLE itself rather than TITLE_SEARCH gives us 
          # results more like SFX4 native, without so many 'also known as' titles confusing
          # things.           
          "(T.TITLE_DISPLAY LIKE '#{connection.quote_string(title_query_param)}%' OR T.TITLE_SORT LIKE '#{connection.quote_string(title_query_param)}%')"
          #"TS.TITLE_SEARCH LIKE '#{connection.quote_string(title_query_param)}%'"
        else # exact
          "( TS.TITLE_SEARCH = '#{connection.quote_string(title_query_param)}' OR 
             T.TITLE_DISPLAY = '#{connection.quote_string(title_query_param)}' OR
             T.TITLE_SORT = '#{connection.quote_string(title_query_param)}'
           )"                        <commit_msg>hacky workaround to direct connect to SFX MySQL db for A-Z lookup, that I don't entirely understand<commit_after>require 'nokogiri'
module SearchMethods
  module Sfx4
    include MetadataHelper # for normalize_lccn

    protected
    # Class method for the module that gets called by the umlaut:load_sfx_urls task.
    # Determines whether we should attempt to fetch SFX urls.
    # Will probably be deprecated in the near future.
    def self.fetch_urls?
      sfx4_base.connection_configured?
    end

    # Class method for the module that gets called by the umlaut:load_sfx_urls task.
    # Kind of hacky way of trying to extract target URLs from SFX4.
    # Will probably be deprecated in the near future.
    def self.fetch_urls
      sfx4_base.fetch_urls
    end
    
    # Class method for the module.
    # Returns the SFX4 base class in order to establish a connection.
    def self.sfx4_base
      # Need to do this convoluted Module.const_get so that we find the
      # correct class. Otherwise the module looks locally and can't find it.
      Module.const_get(:Sfx4).const_get(:Local).const_get(:Base)
    end

    # Instance method that returns the SFX4 AzTitle class for this search method.
    # Can be overridden by search methods that want to include this one.
    def az_title_klass
      # Need to do this convoluted Module.const_get so that we find the
      # correct class. Otherwise the module looks locally and can't find it.
      Module.const_get(:Sfx4).const_get(:Local).const_get(:AzTitle)
    end

    # Instance method that returns the SFX4 DB connection for this search method.
    def sfx4_db_connection
      az_title_klass.connection
    end

    # Needs to return ContextObjects
    def find_by_title
      connection = sfx4_db_connection
      query_match_clause = case search_type_param
        when "contains"
          terms = title_query_param.split(" ")
          #SFX4 seems to ignore 'the' or 'a' on the front, so we will too. 
          if (["the", "a"].include? terms[0])
            terms = terms.slice(1..-1)
          end
          # Then make each term required, but stemmed. Seems to match SFX4, 
          # and more importantly give us decent results. 
          #
          # For reasons we can't entirely tell, the wildcard "*" on terms of less
          # than 2 causes false negatives. Otherwise we use it to be consistent
          # with SFX. This reverse-engineering is full of pitfalls.
          query = terms.collect do |term|
            "+" + connection.quote_string(term) + (term.length > 2 ? "*" : "")
          end.join(" ")
          "MATCH (TS.TITLE_SEARCH) AGAINST ('#{query}' IN BOOLEAN MODE)"
        when "begins"
          # For 'begins', searching against TITLE itself rather than TITLE_SEARCH gives us 
          # results more like SFX4 native, without so many 'also known as' titles confusing
          # things.           
          "(T.TITLE_DISPLAY LIKE '#{connection.quote_string(title_query_param)}%' OR T.TITLE_SORT LIKE '#{connection.quote_string(title_query_param)}%')"
          #"TS.TITLE_SEARCH LIKE '#{connection.quote_string(title_query_param)}%'"
        else # exact
          "( TS.TITLE_SEARCH = '#{connection.quote_string(title_query_param)}' OR 
             T.TITLE_DISPLAY = '#{connection.quote_string(title_query_param)}' OR
             T.TITLE_SORT = '#{connection.quote_string(title_query_param)}'
           )"                        
{'fraction_non_alphanumeric': 0.07514892317091798, 'fraction_numerical': 0.005804185122957079, 'mean_word_length': 2.597802197802198, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '351939', 'n_tokens_mistral': 1939, 'n_tokens_neox': 1876, 'n_words': 654}