<commit_before># Ruby Nickel Library
# Copyright (c) 2008-2011 <NAME>, <EMAIL>, http://hazelmade.com
# MIT License [http://www.opensource.org/licenses/mit-license.php]

require 'logger'
require_relative 'zdate'
require_relative 'ztime'
require_relative 'query_constants'
require_relative 'ruby_ext/calling_method'

module Nickel

  class NLPQuery < String
    include NLPQueryConstants

    # Note there is no initialize here, it is inherited from string class.
    attr_reader :after_formatting, :changed_in, :message

    def standardize
      @query = self.dup # needed for case correcting after extract_message has been called
      query_formatting  # easy text manipulation, no regex involved here
      query_pre_processing  # puts query in the form that construct_finder understands, lots of manipulation here
      self
    end

    def query_formatting
      gsub!(/\n/,'')
      downcase!
      remove_unused_punctuation
      replace_backslashes
      run_spell_check
      remove_unnecessary_words
      standardize_days
      standardize_months
      standardize_numbers
      standardize_am_pm
      replace_hyphens
      insert_repeats_before_words_indicating_recurrence_lame
      insert_space_at_end_of_string_lame
      @after_formatting = self.dup    # save current state
    end

    # Usage:
    #   self.nsub!(/foo/, 'bar')
    #
    # nsub! is like gsub! except it logs the calling method in @changed_in.
    # There is another difference: When using blocks, matched strings are
    # available as block params, e.g.: # nsub!(/(match1)(match2)/) {|m1,m2|}
    #
    # I wrote this because I was having problems overriding gsub and passing
    # a block from the new gsub to super.
    def nsub!(*args)
      if m = self.match(args[0])    # m will now hold the FIRST set of backreferenced matches
        # there is at least one match
        @changed_in ||= []
        @changed_in << calling_method
        if block_given?
          # gsub!(args[0]) {yield(*m.to_a[1..-1])}    # There is a bug here: If gsub matches more than once,
                                                      # then the first set of referenced matches will be passed to the block
          ret_str = m.pre_match + m[0].sub(args[0]) {yield(*m.to_a[1..-1])}   # this will take care of the first set of matches
          while (m_old = m.dup) && (m = m.post_match.match(args[0]))
            ret_str << m.pre_match + m[0].sub(args[0]) {yield(*m.to_a[1..-1])}
          end
          ret_str << m_old.post_match
          self.sub!(/.*/,ret_str)
        else
          gsub!(args[0],args[1])
        end
      end
    end

    def query_pre_processing
      standardize_input
    end

    def remove_unused_punctuation
      nsub!(/,/,' ')
      nsub!(/\./,'')
      nsub!(/;/,'')
      nsub!(/['`]/,'')
    end
<commit_msg>Removed calling_method in favour of using caller directly<commit_after># Ruby Nickel Library
# Copyright (c) 2008-2011 <NAME>, <EMAIL>, http://hazelmade.com
# MIT License [http://www.opensource.org/licenses/mit-license.php]

require 'logger'
require_relative 'zdate'
require_relative 'ztime'
require_relative 'query_constants'

module Nickel

  class NLPQuery < String
    include NLPQueryConstants

    # Note there is no initialize here, it is inherited from string class.
    attr_reader :after_formatting, :changed_in, :message

    def standardize
      @query = self.dup # needed for case correcting after extract_message has been called
      query_formatting  # easy text manipulation, no regex involved here
      query_pre_processing  # puts query in the form that construct_finder understands, lots of manipulation here
      self
    end

    def query_formatting
      gsub!(/\n/,'')
      downcase!
      remove_unused_punctuation
      replace_backslashes
      run_spell_check
      remove_unnecessary_words
      standardize_days
      standardize_months
      standardize_numbers
      standardize_am_pm
      replace_hyphens
      insert_repeats_before_words_indicating_recurrence_lame
      insert_space_at_end_of_string_lame
      @after_formatting = self.dup    # save current state
    end

    # Usage:
    #   self.nsub!(/foo/, 'bar')
    #
    # nsub! is like gsub! except it logs the calling method in @changed_in.
    # There is another difference: When using blocks, matched strings are
    # available as block params, e.g.: # nsub!(/(match1)(match2)/) {|m1,m2|}
    #
    # I wrote this because I was having problems overriding gsub and passing
    # a block from the new gsub to super.
    def nsub!(*args)
      if m = self.match(args[0])    # m will now hold the FIRST set of backreferenced matches
        # there is at least one match
        @changed_in ||= []
        @changed_in << caller[1][/(\w+)\W*$/, 1]
        if block_given?
          # gsub!(args[0]) {yield(*m.to_a[1..-1])}    # There is a bug here: If gsub matches more than once,
                                                      # then the first set of referenced matches will be passed to the block
          ret_str = m.pre_match + m[0].sub(args[0]) {yield(*m.to_a[1..-1])}   # this will take care of the first set of matches
          while (m_old = m.dup) && (m = m.post_match.match(args[0]))
            ret_str << m.pre_match + m[0].sub(args[0]) {yield(*m.to_a[1..-1])}
          end
          ret_str << m_old.post_match
          self.sub!(/.*/,ret_str)
        else
          gsub!(args[0],args[1])
        end
      end
    end

    def query_pre_processing
      standardize_input
    end

    def remove_unused_punctuation
      nsub!(/,/,' ')
      nsub!(/\./,'')
      nsub!(/;/,'')
      nsub!(/['`]/,'')
    end

{'fraction_non_alphanumeric': 0.10063671736823487, 'fraction_numerical': 0.009904492394764769, 'mean_word_length': 2.512422360248447, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1159373', 'n_tokens_mistral': 1793, 'n_tokens_neox': 1740, 'n_words': 506}