<commit_before>require "date"
require "time"
require "treetop"
require_relative "datatype_extras"
require_relative "sql_treetop_load"
require_relative "result_tree_to_hbase_converter"
require_relative "result_tree_to_json_converter"

# This module provides the methods necessary to parse valid SQL
# sentences into:
#
# * Treetop Syntax Trees
# * Hash Trees
# * JSON Trees
# * HBase (Thrift)
#
# It also allows to directly execute valid SQL sentences as Thrift methods.
module HipsterSqlToHbase
  
  # Treetop base SQL parser.
  #
  # Forked from <NAME>'s guillotine and perfected by Jean Lescure.
  #
  # https://www.github.com/smtlaissezfaire/guillotine/tree/master/lib/guillotine/parser
  #
  # https://www.github.com/jeanlescure/hipster_sql_to_hbase
  class SyntaxParser < SQLParser
    class ItemsNode < Treetop::Runtime::SyntaxNode
      def values
        items.values.unshift(item.value)
      end
    end

    class ItemNode < Treetop::Runtime::SyntaxNode
      def values
        [value]
      end
      
      def value
        text_value.to_sym
      end
    end
  end
  
  # This class provides the initial parsed result with the methods
  # necessary to be transformed into HBase (Thrift) or JSON, as well
  # as to be executed as a Thrift method.
  class ResultTree < Hash
    def initialize(hash)
      hash.each { |k,v| self[k] = v }
    end
    
    # Transforms itself into an HBase (Thrift) method.
    # 
    # === example:
    #   rt = HipsterSqlToHbase::ResultTree.new({...})
    #   rt.to_hbase #=> {:method => 'mutateRow', :arguments => {...}}
    def to_hbase
      HipsterSqlToHbase::ResultTreeToHbaseConverter.new().convert self
    end
    
    # Transforms itself into a JSON object.
    # 
    # === example:
    #   user = HipsterSqlToHbase::ResultTree.new({...})
    #   user.to_json #=> {:user => {:user_name => 'bob', :pass => '<PASSWORD>', ...}}
    def to_json
      HipsterSqlToHbase::ResultTreeToJsonConverter.new().convert self
    end
    
    # Executes itself as an HBase (Thrift) method.
    # Short for running .to_hbase and then sending the result to the executor.
    # 
    # === example:
    #   my_call = HipsterSqlToHbase::ResultTree.new({...})
    #   my_call.execute 
    #
    # <em>if no arguments are passed the executor will use any previously set host 
    # and port.</em>
    def execute(host=nil,port=nil)
      self.to_hbase.execute(host,port)
    end
  end
  
  class << self
    # Generate a Treetop syntax tree from a valid, SQL string.
    #
    # === example:
    #   HipsterSqlToHbase.parse_syntax "INSERT INTO users (user,password) VALUES ('user1','<PASSWORD>'),('user2','<PASSWORD>')"
    #
    # === outputs:
    #   #<Class:#<Treetop::Runtime::SyntaxNode:0x3274c98>>
    def parse_syntax(string)
      HipsterSqlToHbase::SyntaxParser.new.parse(string.squish)
    end
    
    # Generate a <b>HipsterSqlToHbase</b>::<b>ResultTree</b> from a valid, SQL string.
    #
    # === example:
    #   HipsterSqlToHbase.parse_tree "SELECT user,password FROM users WHERE id=1"
    #
    # === outputs:
    #   {
    #    :query_type=>:select,
    #    :query_hash=>{
    #      :select=>["user", "password"],
    #      :from=>"users",
    #      :where=>[{:column=>"id", :condition=>:"=", :value=>1}],
    #      :limit=>nil,
    #      :order_by=>nil
    #    }
    #   }
    def parse_tree(string)
      syntax_tree = parse_syntax(string)
      return nil if syntax_tree.nil?
      HipsterSqlToHbase::ResultTree.new({:query_type => syntax_tree.query_type, :query_hash => syntax_tree.tree})
    end
    
    # Generate a <b>HipsterSqlToHbase</b>::<b>ThriftCallGroup</b> from a valid, SQL string.
    #
    # === example:
    #    HipsterSqlToHbase.parse "INSERT INTO `users` (`user`,`pass`) VALUES ('andy','<PASSWORD>'),('<PASSWORD>','<PASSWORD>')"
    #
    # === outputs:
    #    [
    #      {
    #        :method=>"mutateRow", 
    #        :arguments=>[
    #          "users", 
    #          "c6af1d5b-01d7-477c-9539-f35a1e0758e2", 
    #          [<Apache::Hadoop::Hbase::Thrift::Mutation>, <Apache::Hadoop::Hbase::Thrift::Mutation>], 
    #          {}
    #        ]
    #      }, 
    #      {<commit_msg>parse_hash added for convenience.<commit_after>require "date"
require "time"
require "treetop"
require_relative "datatype_extras"
require_relative "sql_treetop_load"
require_relative "result_tree_to_hbase_converter"
require_relative "result_tree_to_json_converter"

# This module provides the methods necessary to parse valid SQL
# sentences into:
#
# * Treetop Syntax Trees
# * Hash Trees
# * JSON Trees
# * HBase (Thrift)
#
# It also allows to directly execute valid SQL sentences as Thrift methods.
module HipsterSqlToHbase
  
  # Treetop base SQL parser.
  #
  # Forked from <NAME>'s guillotine and perfected by Jean Lescure.
  #
  # https://www.github.com/smtlaissezfaire/guillotine/tree/master/lib/guillotine/parser
  #
  # https://www.github.com/jeanlescure/hipster_sql_to_hbase
  class SyntaxParser < SQLParser
    class ItemsNode < Treetop::Runtime::SyntaxNode
      def values
        items.values.unshift(item.value)
      end
    end

    class ItemNode < Treetop::Runtime::SyntaxNode
      def values
        [value]
      end
      
      def value
        text_value.to_sym
      end
    end
  end
  
  # This class provides the initial parsed result with the methods
  # necessary to be transformed into HBase (Thrift) or JSON, as well
  # as to be executed as a Thrift method.
  class ResultTree < Hash
    def initialize(hash)
      hash.each { |k,v| self[k] = v }
    end
    
    # Transforms itself into an HBase (Thrift) method.
    # 
    # === example:
    #   rt = HipsterSqlToHbase::ResultTree.new({...})
    #   rt.to_hbase #=> {:method => 'mutateRow', :arguments => {...}}
    def to_hbase
      HipsterSqlToHbase::ResultTreeToHbaseConverter.new().convert self
    end
    
    # Transforms itself into a JSON object.
    # 
    # === example:
    #   user = HipsterSqlToHbase::ResultTree.new({...})
    #   user.to_json #=> {:user => {:user_name => 'bob', :pass => '<PASSWORD>', ...}}
    def to_json
      HipsterSqlToHbase::ResultTreeToJsonConverter.new().convert self
    end
    
    # Executes itself as an HBase (Thrift) method.
    # Short for running .to_hbase and then sending the result to the executor.
    # 
    # === example:
    #   my_call = HipsterSqlToHbase::ResultTree.new({...})
    #   my_call.execute 
    #
    # <em>if no arguments are passed the executor will use any previously set host 
    # and port.</em>
    def execute(host=nil,port=nil)
      self.to_hbase.execute(host,port)
    end
  end
  
  class << self
    # Generate a Treetop syntax tree from a valid, SQL string.
    #
    # === example:
    #   HipsterSqlToHbase.parse_syntax "INSERT INTO users (user,password) VALUES ('user1','<PASSWORD>'),('user2','<PASSWORD>')"
    #
    # === outputs:
    #   #<Class:#<Treetop::Runtime::SyntaxNode:0x3274c98>>
    def parse_syntax(string)
      HipsterSqlToHbase::SyntaxParser.new.parse(string.squish)
    end
    
    # Generate a Hash from a valid, SQL string.
    #
    # === example:
    #   HipsterSqlToHbase.parse_hash "SELECT user,password FROM users WHERE id=1"
    #
    # === outputs:
    #   {
    #    :query_type=>:select,
    #    :query_hash=>{
    #      :select=>["user", "password"],
    #      :from=>"users",
    #      :where=>[{:column=>"id", :condition=>:"=", :value=>1}],
    #      :limit=>nil,
    #      :order_by=>nil
    #    }
    #   }
    def parse_hash(string)
      syntax_tree = parse_syntax(string)
      return nil if syntax_tree.nil?
      { 
        :query_type => syntax_tree.query_type, 
        :query_hash => syntax_tree.tree
      }
    end
    
    # Generate a <b>HipsterSqlToHbase</b>::<b>ResultTree</b> from a valid, SQL string.
    #
    # === example:
    #   HipsterSqlToHbase.parse_tree "SELECT user,password FROM users WHERE id=1"
    #
    # === outputs:
    #   {
    #    :query_type=>:select,
    #    :query_hash=>{
    #      :select=>["user", "password"],
    #      :from=>"users",
    #      :where=>[{:column=>"id", :condition=>:"=", :value=>1}],
    #      :limit=>nil,
    #      :order_by=>nil
    #    }
    #   }
    #
    # === Note: the main difference between **parse_tree** and **parse_hash** is that a **ResultTree** can be further converted to Thrift calls while a **Hash** cannot.
    def parse_tree(string)
      HipsterSqlToHbase::ResultTree.new(parse_hash(string))
    end
    
    # Generate a <b>HipsterSqlToHbase</b>::<b>ThriftCallGroup</b> from a valid, SQL string.
    #
    # === example:
    #    HipsterSqlToHbase.parse "INSERT INTO `users` (`user`,`pass`) VALUES ('andy','<PASSWORD>'),('z<PASSWORD>','<PASSWORD>')"
    #
    # === outputs:
    #    [
    #      {
    #        :method=>"mutateRow", 
    #        :arguments=>[
    #          "users", 
    #          "c6af1d5b-01d7-477c-9539-f35a1e0758e2", 
    #          [<Apache::Hadoop::Hbase::Thrift::Mutation>, <Apache::Hadoop::Hbase::Thrift::Mutation>], 
    #          {}
    #        ]
    #      }, 
    #      {
{'fraction_non_alphanumeric': 0.13822656850895507, 'fraction_numerical': 0.007251950335128008, 'mean_word_length': 2.6480961923847697, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 97, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4122228', 'n_tokens_mistral': 3191, 'n_tokens_neox': 3089, 'n_words': 654}