<commit_before>#!/usr/bin/env ruby
#---
# Author::  <NAME>
# Created:: <2010-06-22 16:35:16>
#+++
require File.join(File.dirname(__FILE__), "common")
require 'optparse'

request = {
  :collection => nil,
  :document_count => 0
}

opts = OptionParser.new do |opts|
  opts.banner = "Usage: ruby #{__FILE__} [OPTIONS] command collection"

  opts.separator ''

  opts.on("--document-count COUNT", "-n", Integer,
          "Document count limit (for debug)") do |n|
    request[:document_count] = n
  end

  opts.on_tail("--help", "-h", "Print this help") do
    puts opts
    exit
  end
end

opts.parse!

command, request[:collection] = ARGV

if command.nil?
  STDERR.puts "Require command."
  exit 1
end

conn = create_connection
response = conn.call("commands/#{command}", request)

if response.nil? or !response["header"]["success"]
  STDERR.puts "ERROR!"
  if response["errors"]
    STDERR.puts response["errors"].join("\n")
  end
  exit 1
elsif response["warnings"]
  puts "WARNING!"
  puts response["warnings"].join("\n")
else
  puts "Sent!"
end<commit_msg>support rebuild_from_scd in cmd_sender<commit_after>#!/usr/bin/env ruby
#---
# Author::  <NAME>
# Created:: <2010-06-22 16:35:16>
#+++
require File.join(File.dirname(__FILE__), "common")
require 'optparse'

request = {
  :collection => nil,
  :document_count => 0
}

opts = OptionParser.new do |opts|
  opts.banner = "Usage: ruby #{__FILE__} [OPTIONS] command collection"

  opts.separator ''

  opts.on("--document-count COUNT", "-n", Integer,
          "Document count limit (for debug)") do |n|
    request[:document_count] = n
  end

  opts.on_tail("--help", "-h", "Print this help") do
    puts opts
    exit
  end
end

opts.parse!

command, request[:collection] = ARGV

if command.nil?
  STDERR.puts "Require command."
  exit 1
end

conn = create_connection
api = "commands/#{command}"
if command=="rebuild"
  api = "collection/rebuild_from_scd"
end
response = conn.call(api, request)

if response.nil? or !response["header"]["success"]
  STDERR.puts "ERROR!"
  if response["errors"]
    STDERR.puts response["errors"].join("\n")
  end
  exit 1
elsif response["warnings"]
  puts "WARNING!"
  puts response["warnings"].join("\n")
else
  puts "Sent!"
end
{'fraction_non_alphanumeric': 0.16734325665313488, 'fraction_numerical': 0.015336039693279206, 'mean_word_length': 3.9730941704035874, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3900946', 'n_tokens_mistral': 813, 'n_tokens_neox': 775, 'n_words': 197}