<commit_before>module Gql::Grammar
  module Functions
    module Legacy

      def replace_gql_with_ruby_brackets(attr_name)
        if attr_name.include?('[')
          attr_name.strip!
          attr_name.gsub!('[','(')
          attr_name.gsub!(']',')')
          attr_name
        else
          attr_name.tap(&:strip!)
        end
      end

      def ALL(*keys)
        scope.all_converters
      end

      def FILTER(converters, filter_name, scope)
        inst_eval = replace_gql_with_ruby_brackets(filter_name.first)
        flatten_uniq(converters.tap(&:flatten!).select{|c| c.query.instance_eval(inst_eval) })
      end

      # Runs the gquery with given key for the present year only.
      #
      # gquery_key - The gquery lookup key. 
      #
      # Returns the result of the gquery for only the present year. 
      #
      # Examples
      #
      #   QUERY_PRESENT(graph_year)   # => 2010
      #
      def QUERY_PRESENT(gquery_key)
        scope.gql.present.subquery(gquery_key.to_s)
      end

      # Runs the gquery with given key for the future year only.
      #
      # gquery_key - The gquery lookup key. 
      #
      # Returns the result of the gquery for only the future year. 
      #
      # Examples
      #
      #   QUERY_FUTURE(graph_year)    # => 2050
      #
      def QUERY_FUTURE(gquery_key)
        scope.gql.future.subquery(gquery_key.to_s)
      end

      def CHILDREN(*converters)
        flatten_uniq(converters.tap(&:flatten!).map{|c| c.converter.children})
      end

      def PARENTS(*converters)
        flatten_uniq(converters.tap(&:flatten!).map{|c| c.converter.parents})
      end
  
      def INTERSECTION(*keys)
        keys.first.flatten & keys.last.flatten
      end

      def EXCLUDE(*keys)
        keys.first.flatten - keys.last.flatten
      end

      def INVALID_TO_ZERO(*keys)
        is_invalid = Proc.new {|v| v.nil? || (v.respond_to?(:nan?) && v.nan?) }

        if values.respond_to?(:map)
          values.tap(&:flatten!).map!{|v| is_invalid.call(v) ? 0 : v }
          values
        else
          is_invalid.call(values) ? 0 : v
        end
      end

      def MAX(*values)
        flatten_compact(values).max
      end
<commit_msg>fix EXCLUDE<commit_after>module Gql::Grammar
  module Functions
    module Legacy

      def replace_gql_with_ruby_brackets(attr_name)
        if attr_name.include?('[')
          attr_name.strip!
          attr_name.gsub!('[','(')
          attr_name.gsub!(']',')')
          attr_name
        else
          attr_name.tap(&:strip!)
        end
      end

      def ALL(*keys)
        scope.all_converters
      end

      def FILTER(converters, filter_name, scope)
        inst_eval = replace_gql_with_ruby_brackets(filter_name.first)
        flatten_uniq(converters.tap(&:flatten!).select{|c| c.query.instance_eval(inst_eval) })
      end

      # Runs the gquery with given key for the present year only.
      #
      # gquery_key - The gquery lookup key. 
      #
      # Returns the result of the gquery for only the present year. 
      #
      # Examples
      #
      #   QUERY_PRESENT(graph_year)   # => 2010
      #
      def QUERY_PRESENT(gquery_key)
        scope.gql.present.subquery(gquery_key.to_s)
      end

      # Runs the gquery with given key for the future year only.
      #
      # gquery_key - The gquery lookup key. 
      #
      # Returns the result of the gquery for only the future year. 
      #
      # Examples
      #
      #   QUERY_FUTURE(graph_year)    # => 2050
      #
      def QUERY_FUTURE(gquery_key)
        scope.gql.future.subquery(gquery_key.to_s)
      end

      def CHILDREN(*converters)
        flatten_uniq(converters.tap(&:flatten!).map{|c| c.converter.children})
      end

      def PARENTS(*converters)
        flatten_uniq(converters.tap(&:flatten!).map{|c| c.converter.parents})
      end
  
      def INTERSECTION(*keys)
        keys.first.flatten & keys.last.flatten
      end

      def EXCLUDE(first, last)
        first.flatten - last.flatten
      end

      def INVALID_TO_ZERO(*keys)
        is_invalid = Proc.new {|v| v.nil? || (v.respond_to?(:nan?) && v.nan?) }

        if values.respond_to?(:map)
          values.tap(&:flatten!).map!{|v| is_invalid.call(v) ? 0 : v }
          values
        else
          is_invalid.call(values) ? 0 : v
        end
      end

      def MAX(*values)
        flatten_compact(values).max
      end

{'fraction_non_alphanumeric': 0.10698096101541252, 'fraction_numerical': 0.004533091568449683, 'mean_word_length': 1.987813134732566, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7047374', 'n_tokens_mistral': 1580, 'n_tokens_neox': 1468, 'n_words': 290}