<commit_before>require 'thor'
require 'norikra/client'

require 'norikra/client/cli/parser'
require 'norikra/client/cli/formatter'

class Norikra::Client
  module CLIUtil
    def client(options)
      Norikra::Client.new(options[:host], options[:port])
    end
  end

  class Target < Thor
    include Norikra::Client::CLIUtil

    desc "list", "show list of targets"
    option :simple, :type => :boolean, :default => false, :desc => "suppress header/footer", :aliases => "-s"
    def list
      puts "TARGET" unless options[:simple]
      targets = client(parent_options).targets
      targets.each do |t|
        puts t
      end
      puts "#{targets.size} targets found." unless options[:simple]
    end

    desc "open TARGET [fieldname1:type1 [fieldname2:type2 [fieldname3:type3] ...]]", "create new target (and define its fields)"
    def open(target, *field_defs)
      fields = nil
      if field_defs.size > 0
        fields = {}
        field_defs.each do |str|
          fname,ftype = str.split(':')
          fields[fname] = ftype
        end
      end
      client(parent_options).open(target, fields)
    end

    desc "close TARGET", "close existing target and all its queries"
    def close(target)
      client(parent_options).close(target)
    end
  end

  class Query < Thor
    include Norikra::Client::CLIUtil

    desc "list", "show list of queries"
    option :simple, :type => :boolean, :default => false, :desc => "suppress header/footer", :aliases => "-s"
    def list
      puts "TARGET\tQUERY_NAME\tQUERY" unless options[:simple]
      queries = client(parent_options).queries
      queries.sort{|a,b| (a['target'] <=> b['target']).nonzero? || a['name'] <=> b['name']}.each do |q|
        puts "#{q['target']}\t#{q['name']}\t#{q['expression']}"
      end
      puts "#{queries.size} queries found." unless options[:simple]
    end

    desc "add QUERY_NAME QUERY_EXPRESSION", "register a query"
    def add(query_name, expression)
      client(parent_options).register(query_name, expression)
    end

    desc "remove QUERY_NAME", "deregister a query"
    def remove(query_name)
      client(parent_options).deregister(query_name)
    end
  end

  class Field < Thor
    include Norikra::Client::CLIUtil

    desc "list TARGET", "show list of field definitions of specified target"
    option :simple, :type => :boolean, :default => false, :desc => "suppress header/footer", :aliases => "-s"
    def list(target)
      puts "FIELD\tTYPE\tOPTIONAL" unless options[:simple]
      fields = client(parent_options).fields
      fields.each do |f|
        puts "#{f['name']}\t#{f['type']}\t#{f['optional']}"
      end
      puts "#{fields.size} fields found." unless options[:simple]
    end

    desc "add TARGET FIELDNAME TYPE", "reserve fieldname and its type of target"
    def add(target, field, type)
      client(parent_options).reserve(target, field, type)
    end
  end

  class Event < Thor
    include Norikra::Client::CLIUtil

    desc "send TARGET", "send data into targets"
    option :format, :type => :string, :default => 'json', :desc => "format of input data per line of stdin [json(default), ltsv]"
    option :batch_size, :type => :numeric, :default => 10000, :desc => "records sent in once transferring (default: 10000)"
    def send(target)
      client = client(parent_options)
      parser = parser(options[:format])
      buffer = []
      $stdin.each_line do |line|<commit_msg>fix bug of missing argument<commit_after>require 'thor'
require 'norikra/client'

require 'norikra/client/cli/parser'
require 'norikra/client/cli/formatter'

class Norikra::Client
  module CLIUtil
    def client(options)
      Norikra::Client.new(options[:host], options[:port])
    end
  end

  class Target < Thor
    include Norikra::Client::CLIUtil

    desc "list", "show list of targets"
    option :simple, :type => :boolean, :default => false, :desc => "suppress header/footer", :aliases => "-s"
    def list
      puts "TARGET" unless options[:simple]
      targets = client(parent_options).targets
      targets.each do |t|
        puts t
      end
      puts "#{targets.size} targets found." unless options[:simple]
    end

    desc "open TARGET [fieldname1:type1 [fieldname2:type2 [fieldname3:type3] ...]]", "create new target (and define its fields)"
    def open(target, *field_defs)
      fields = nil
      if field_defs.size > 0
        fields = {}
        field_defs.each do |str|
          fname,ftype = str.split(':')
          fields[fname] = ftype
        end
      end
      client(parent_options).open(target, fields)
    end

    desc "close TARGET", "close existing target and all its queries"
    def close(target)
      client(parent_options).close(target)
    end
  end

  class Query < Thor
    include Norikra::Client::CLIUtil

    desc "list", "show list of queries"
    option :simple, :type => :boolean, :default => false, :desc => "suppress header/footer", :aliases => "-s"
    def list
      puts "TARGET\tQUERY_NAME\tQUERY" unless options[:simple]
      queries = client(parent_options).queries
      queries.sort{|a,b| (a['target'] <=> b['target']).nonzero? || a['name'] <=> b['name']}.each do |q|
        puts "#{q['target']}\t#{q['name']}\t#{q['expression']}"
      end
      puts "#{queries.size} queries found." unless options[:simple]
    end

    desc "add QUERY_NAME QUERY_EXPRESSION", "register a query"
    def add(query_name, expression)
      client(parent_options).register(query_name, expression)
    end

    desc "remove QUERY_NAME", "deregister a query"
    def remove(query_name)
      client(parent_options).deregister(query_name)
    end
  end

  class Field < Thor
    include Norikra::Client::CLIUtil

    desc "list TARGET", "show list of field definitions of specified target"
    option :simple, :type => :boolean, :default => false, :desc => "suppress header/footer", :aliases => "-s"
    def list(target)
      puts "FIELD\tTYPE\tOPTIONAL" unless options[:simple]
      fields = client(parent_options).fields(target)
      fields.each do |f|
        puts "#{f['name']}\t#{f['type']}\t#{f['optional']}"
      end
      puts "#{fields.size} fields found." unless options[:simple]
    end

    desc "add TARGET FIELDNAME TYPE", "reserve fieldname and its type of target"
    def add(target, field, type)
      client(parent_options).reserve(target, field, type)
    end
  end

  class Event < Thor
    include Norikra::Client::CLIUtil

    desc "send TARGET", "send data into targets"
    option :format, :type => :string, :default => 'json', :desc => "format of input data per line of stdin [json(default), ltsv]"
    option :batch_size, :type => :numeric, :default => 10000, :desc => "records sent in once transferring (default: 10000)"
    def send(target)
      client = client(parent_options)
      parser = parser(options[:format])
      buffer = []
      $stdin.each_line do |line|
{'fraction_non_alphanumeric': 0.135504507124164, 'fraction_numerical': 0.004943297470194824, 'mean_word_length': 3.25417439703154, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1748480', 'n_tokens_mistral': 2186, 'n_tokens_neox': 2144, 'n_words': 609}