<commit_before>require 'benchmark'
require 'CSV'
require 'impas-client'

client = Impas::Client.new

if ARGV.length != 2
  puts "Usage:"
  puts "ruby parse_tsv.rb [Parse target file] [Impas group key]"
  exit
end

puts "Parse start. Target:#{ARGV[0]}"

puts Benchmark::CAPTION
puts Benchmark.measure{
  CSV.foreach(ARGV[0], :col_sep=>"\t") do |row|
    client.add_url ARGV[1], row[1], row[0]
  end
}
<commit_msg>サンプルプログラム修正<commit_after>require 'benchmark'
require 'CSV'
require 'impas-client'
require 'parallel'

client = Impas::Client.new({api_url:'http://impas-hideack.sqale.jp/'})

if ARGV.length != 3
  puts "Usage:"
  puts "ruby parse_tsv.rb [Parse target file] [Impas group key] [client threads]"
  exit
end

puts "CSV Parse start. Target:#{ARGV[0]}"

targetUrls = []
CSV.foreach(ARGV[0], :col_sep=>"\t") do |row|
  targetUrls << row
end

puts "CSV Parse finished."
puts "Impas post start."

puts Benchmark::CAPTION
puts Benchmark.measure{
  Parallel.each(targetUrls, in_threads: ARGV[2].to_i) do |row|
    client.add_url ARGV[1], row[1], row[0]
  end
}

{'fraction_non_alphanumeric': 0.15739868049010367, 'fraction_numerical': 0.012252591894439209, 'mean_word_length': 4.835164835164835, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6396591', 'n_tokens_mistral': 445, 'n_tokens_neox': 435, 'n_words': 104}