<commit_before>require 'nickel/zdate'
require 'nickel/ztime'
require 'nickel/nlp_query_constants'
require 'holidays'


module Nickel
  class NLPQuery
    include NLPQueryConstants


    def initialize(query_str, input_date)
      @query_str = query_str.dup
      @curdate = input_date
      @holidays = []

      # build a @holidays array of all holidays for both the US and the passed region
      # initially load all relevant regions - there may be a future problem for holidays
      # with the same name that have different dates by region

      Holidays.between(input_date.to_date,input_date.add_days(365).to_date, 'us', 'ca', 'es', :informal).each do |holiday|
        # clean names for nlp matching
        holiday[:name].gsub!(/,/, ' ')
        holiday[:name].gsub!(/\./, '')
        holiday[:name].gsub!(/;/, '')
        holiday[:name].gsub!(/['`]/, '')
        name = holiday[:name].downcase
        name.gsub!(/new years day/,'new years')
        name.gsub!(/christmas day/,'christmas')
        # add cleaned names to @holidays array
        @holidays.push({:date => holiday[:date], :name => name})
      end
    end

    attr_reader :after_formatting, :changed_in

    def standardize
      @query = query_str.dup # needed for case correcting after extract_message has been called
      query_formatting  # easy text manipulation, no regex involved here
      query_pre_processing  # puts query in the form that construct_finder understands, lots of manipulation here
      query_str.dup
    end

    def query_formatting
      query_str.gsub!(/\n/, '')
      query_str.downcase!
      remove_unused_punctuation
      replace_backslashes
      run_spell_check
      remove_unnecessary_words
      standardize_days
      standardize_months
      standardize_numbers
      standardize_times
      standardize_am_pm
      replace_hyphens
      remove_commas_inside_dates
      convert_holidays_to_dates
#      insert_repeats_before_words_indicating_recurrence_lame
      insert_space_at_end_of_string_lame
      @after_formatting = query_str.dup    # save current state
    end

    # Usage:
    #   self.nsub!(/foo/, 'bar')
    #
    # nsub! is like gsub! except it logs the calling method in @changed_in.
    # There is another difference: When using blocks, matched strings are
    # available as block params, e.g.: # nsub!(/(match1)(match2)/) {|m1,m2|}
    #
    # I wrote this because I was having problems overriding gsub and passing
    # a block from the new gsub to super.
    def nsub!(*args)
      if m = query_str.match(args[0])    # m will now hold the FIRST set of backreferenced matches
        # there is at least one match
        @changed_in ||= []
        @changed_in << caller[1][/(\w+)\W*$/, 1]
        if block_given?
          # query_str.gsub!(args[0]) {yield(*m.to_a[1..-1])}    # There is a bug here: If gsub matches more than once,
                                                      # then the first set of referenced matches will be passed to the block
          ret_str = m.pre_match + m[0].sub(args[0]) { yield(*m.to_a[1..-1]) }   # this will take care of the first set of matches
          while (m_old = m.dup) && (m = m.post_match.match(args[0]))
            ret_str << m.pre_match + m[0].sub(args[0]) { yield(*m.to_a[1..-1]) }
          end
          ret_str << m_old.post_match
          query_str.sub!(/.*/, ret_str)
        else
          query_str.gsub!(args[0], args[1])
        end
      end
    end

    def query_pre_processing
      standardize_input
    end

    def remove_unused_punctuation
      nsub!(/\s*,\s*/, ' , ')                  # make , into a token - useful to interpret conjunctive modifiers
      nsub!(/\./, '')
      nsub!(/;/, '')
      nsub!(/['`]/, '')
      nsub!(/[\)\(]/, '')
    end

    def replace_backslashes
      nsub!(/\\/, '/')
    end

    def run_spell_check
      nsub!(/tomm?orr?ow|romorrow/, 'tomorrow')
      nsub!(/weeknd/, 'weekend')
      nsub!(/weekends/, 'every sat sun')
      nsub!(/everyother/, 'every other')
      nsub!(/w\/o/, 'week of')
      nsub!(/weak/, 'week')
      nsub!(/everyweek/, 'every week')
      nsub!(/everymonth/, 'every month')
      nsub!(/c?h[oa]nn?[aui][ck][ck]?[ua]h?/, 'hannukkah')
      nsub!(/frist/, '1st')
      nsub!(/eveyr|evrey/, 'every')
      nsub!(/fridya|friady|fridy/, 'friday')
      nsub!(/thurdsday/, 'thursday')
      nsub!(/x-?mas/, 'christmas')
      nsub!(/st\s+(patrick|patty|pat)s?(\s+day)?/, 'st patricks day')
      nsub!(/frouth/, 'fourth')
      nsub!(/\btill\b/, 'through')
      nsub!(/\bthru\b|\bthrouh\b|\bthough\b|\bthrew\b|\bthrow\b|\bthroug\b|\bthuogh\b/, 'through')
      nsub!(/weekdays|every\s+weekday/, 'every monday through friday')
      nsub!(/\bevery?day\b/, 'every day')
      nsub!(/eigth/, 'eighth')
      nsub!(/bi[-\s]monthly/, 'bimonthly')
      nsub!(/tri[-\s]monthly/, 'trimonthly')
    end

    def remove_unnecessary_words
      nsub!(/this coming/, 'thiscoming')        #sm - to mean the next occurence
      nsub!(/the day after next/, 'thedayafter tomorrow')     #sm -
      nsub!(/the week after next/, 'theweekafternext')     #sm -
      nsub!(/the following week/, 'thefollowingweek')
      nsub!(/the following month/, 'thefollowingmonth')
      nsub!(/the day after/, 'thedayafter')     #sm - when used with 'tomorrow' or a date
      nsub!(/the week after/, 'theweekafter')   #sm - when used with a date
      nsub!(/the month after/, 'themonthafter') #sm - when used with a date
      nsub!(/the day before/, 'thedaybefore')   #sm - use with a date
      nsub!(/the week before/, 'theweekbefore')   #sm - use with a date
      nsub!(/the month before/, 'theweekbefore')   #sm - use with a date
      nsub!(/(a|any)\s*(day|days)\s+(before)/, 'anydaybefore')    #sm
      nsub!(/(a|any)\s*(day|days)\s+(after)/, 'anydayafter')    #sm
      nsub!(/o'?clock/, '')
      nsub!(/\btom\b/, 'tomorrow')
      nsub!(/\bmi(dn|nd)ight\b/, '11:59pm')
      nsub!(/final/, 'thelast')                 # sm - need to distinguish between final and prior - "thelast" means final
      nsub!(/the last/, 'thelast')              # sm - if the token is just "last" then it will be interpretted as prior
      nsub!(/recur(s|r?ing)?/, 'repeats')
      nsub!(/\beach\b/, 'every')
      nsub!(/running\s+(until|through)/, 'through')
      nsub!(/runn?(s|ing)|go(ing|e?s)/, 'for')
      nsub!(/next\s+occ?urr?[ae]nce(\s+is)?/, 'start')
      nsub!(/next\s+date(\s+it)?(\s+occ?urr?s)?(\s+is)?/, 'start')
      nsub!(/forever/, 'repeats daily')
      nsub!(/\bany\s*day\b/, '')
      nsub!(/^anytime$/, 'every day')  # user entered anytime by itself, not 'dayname anytime', caught next
      nsub!(/any(\s)?time|whenever/, 'all day')
      nsub!(/\bnoon\b/, '12:00pm')
    end

    def standardize_days
      nsub!(/mondays/, 'every mon')
      nsub!(/monday/, 'mon')<commit_msg>Bug fix - removed "anytime"<commit_after>require 'nickel/zdate'
require 'nickel/ztime'
require 'nickel/nlp_query_constants'
require 'holidays'


module Nickel
  class NLPQuery
    include NLPQueryConstants


    def initialize(query_str, input_date)
      @query_str = query_str.dup
      @curdate = input_date
      @holidays = []

      # build a @holidays array of all holidays for both the US and the passed region
      # initially load all relevant regions - there may be a future problem for holidays
      # with the same name that have different dates by region

      Holidays.between(input_date.to_date,input_date.add_days(365).to_date, 'us', 'ca', 'es', :informal).each do |holiday|
        # clean names for nlp matching
        holiday[:name].gsub!(/,/, ' ')
        holiday[:name].gsub!(/\./, '')
        holiday[:name].gsub!(/;/, '')
        holiday[:name].gsub!(/['`]/, '')
        name = holiday[:name].downcase
        name.gsub!(/new years day/,'new years')
        name.gsub!(/christmas day/,'christmas')
        # add cleaned names to @holidays array
        @holidays.push({:date => holiday[:date], :name => name})
      end
    end

    attr_reader :after_formatting, :changed_in

    def standardize
      @query = query_str.dup # needed for case correcting after extract_message has been called
      query_formatting  # easy text manipulation, no regex involved here
      query_pre_processing  # puts query in the form that construct_finder understands, lots of manipulation here
      query_str.dup
    end

    def query_formatting
      query_str.gsub!(/\n/, '')
      query_str.downcase!
      remove_unused_punctuation
      replace_backslashes
      run_spell_check
      remove_unnecessary_words
      standardize_days
      standardize_months
      standardize_numbers
      standardize_times
      standardize_am_pm
      replace_hyphens
      remove_commas_inside_dates
      convert_holidays_to_dates
#      insert_repeats_before_words_indicating_recurrence_lame
      insert_space_at_end_of_string_lame
      @after_formatting = query_str.dup    # save current state
    end

    # Usage:
    #   self.nsub!(/foo/, 'bar')
    #
    # nsub! is like gsub! except it logs the calling method in @changed_in.
    # There is another difference: When using blocks, matched strings are
    # available as block params, e.g.: # nsub!(/(match1)(match2)/) {|m1,m2|}
    #
    # I wrote this because I was having problems overriding gsub and passing
    # a block from the new gsub to super.
    def nsub!(*args)
      if m = query_str.match(args[0])    # m will now hold the FIRST set of backreferenced matches
        # there is at least one match
        @changed_in ||= []
        @changed_in << caller[1][/(\w+)\W*$/, 1]
        if block_given?
          # query_str.gsub!(args[0]) {yield(*m.to_a[1..-1])}    # There is a bug here: If gsub matches more than once,
                                                      # then the first set of referenced matches will be passed to the block
          ret_str = m.pre_match + m[0].sub(args[0]) { yield(*m.to_a[1..-1]) }   # this will take care of the first set of matches
          while (m_old = m.dup) && (m = m.post_match.match(args[0]))
            ret_str << m.pre_match + m[0].sub(args[0]) { yield(*m.to_a[1..-1]) }
          end
          ret_str << m_old.post_match
          query_str.sub!(/.*/, ret_str)
        else
          query_str.gsub!(args[0], args[1])
        end
      end
    end

    def query_pre_processing
      standardize_input
    end

    def remove_unused_punctuation
      nsub!(/\s*,\s*/, ' , ')                  # make , into a token - useful to interpret conjunctive modifiers
      nsub!(/\./, '')
      nsub!(/;/, '')
      nsub!(/['`]/, '')
      nsub!(/[\)\(]/, '')
    end

    def replace_backslashes
      nsub!(/\\/, '/')
    end

    def run_spell_check
      nsub!(/tomm?orr?ow|romorrow/, 'tomorrow')
      nsub!(/weeknd/, 'weekend')
      nsub!(/weekends/, 'every sat sun')
      nsub!(/everyother/, 'every other')
      nsub!(/w\/o/, 'week of')
      nsub!(/weak/, 'week')
      nsub!(/everyweek/, 'every week')
      nsub!(/everymonth/, 'every month')
      nsub!(/c?h[oa]nn?[aui][ck][ck]?[ua]h?/, 'hannukkah')
      nsub!(/frist/, '1st')
      nsub!(/eveyr|evrey/, 'every')
      nsub!(/fridya|friady|fridy/, 'friday')
      nsub!(/thurdsday/, 'thursday')
      nsub!(/x-?mas/, 'christmas')
      nsub!(/st\s+(patrick|patty|pat)s?(\s+day)?/, 'st patricks day')
      nsub!(/frouth/, 'fourth')
      nsub!(/\btill\b/, 'through')
      nsub!(/\bthru\b|\bthrouh\b|\bthough\b|\bthrew\b|\bthrow\b|\bthroug\b|\bthuogh\b/, 'through')
      nsub!(/weekdays|every\s+weekday/, 'every monday through friday')
      nsub!(/\bevery?day\b/, 'every day')
      nsub!(/eigth/, 'eighth')
      nsub!(/bi[-\s]monthly/, 'bimonthly')
      nsub!(/tri[-\s]monthly/, 'trimonthly')
    end

    def remove_unnecessary_words
      nsub!(/this coming/, 'thiscoming')        #sm - to mean the next occurence
      nsub!(/the day after next/, 'thedayafter tomorrow')     #sm -
      nsub!(/the week after next/, 'theweekafternext')     #sm -
      nsub!(/the following week/, 'thefollowingweek')
      nsub!(/the following month/, 'thefollowingmonth')
      nsub!(/the day after/, 'thedayafter')     #sm - when used with 'tomorrow' or a date
      nsub!(/the week after/, 'theweekafter')   #sm - when used with a date
      nsub!(/the month after/, 'themonthafter') #sm - when used with a date
      nsub!(/the day before/, 'thedaybefore')   #sm - use with a date
      nsub!(/the week before/, 'theweekbefore')   #sm - use with a date
      nsub!(/the month before/, 'theweekbefore')   #sm - use with a date
      nsub!(/(a|any)\s*(day|days)\s+(before)/, 'anydaybefore')    #sm
      nsub!(/(a|any)\s*(day|days)\s+(after)/, 'anydayafter')    #sm
      nsub!(/o'?clock/, '')
      nsub!(/\btom\b/, 'tomorrow')
      nsub!(/\bmi(dn|nd)ight\b/, '11:59pm')
      nsub!(/final/, 'thelast')                 # sm - need to distinguish between final and prior - "thelast" means final
      nsub!(/the last/, 'thelast')              # sm - if the token is just "last" then it will be interpretted as prior
      nsub!(/recur(s|r?ing)?/, 'repeats')
      nsub!(/\beach\b/, 'every')
      nsub!(/running\s+(until|through)/, 'through')
      nsub!(/runn?(s|ing)|go(ing|e?s)/, 'for')
      nsub!(/next\s+occ?urr?[ae]nce(\s+is)?/, 'start')
      nsub!(/next\s+date(\s+it)?(\s+occ?urr?s)?(\s+is)?/, 'start')
      nsub!(/forever/, 'repeats daily')
      nsub!(/\bany\s*day\b/, '')
#      nsub!(/^anytime$/, 'every day')  # user entered anytime by itself, not 'dayname anytime', caught next
#      nsub!(/any(\s)?time|whenever/, 'all day')
      nsub!(/\bnoon\b/, '12:00pm')
    end

    def standardize_days
      nsub!(/mondays/, 'every mon')
      nsub!(/monday/, 'mon')
{'fraction_non_alphanumeric': 0.15742227412056634, 'fraction_numerical': 0.00481681506349438, 'mean_word_length': 2.4832231825114386, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7355747', 'n_tokens_mistral': 4753, 'n_tokens_neox': 4679, 'n_words': 1118}