<commit_before><commit_msg>Initial version of netbios counter.  Returns counts of the top company and netbios names in json format.<commit_after>require 'oj'
require 'optparse'

options = Hash.new(0)
OptionParser.new do |opts|
  opts.banner = "Usage: example.rb [options]"

  opts.on("-c", "--count [INT]", OptionParser::DecimalInteger, 
          "Specify the number of top results") do |count|
    options[:top_count] = count
  end
end.parse!

NUM_TOP_RECORDS = options[:top_count] > 1 ? options[:top_count] : 5

module NameCounter
  def count(hash)
    name = hash[field].to_s
    @names[name] += 1 unless (name.empty? || name == 'UNKNOWN')
  end

  def top_names
    [].tap do |counts|
      ordered_names.to_a.take(NUM_TOP_RECORDS).each do |values|
        counts << { name: values[0], count: values[1] }
      end
    end
  end

  def ordered_names
    Hash[@names.sort_by{|k, v| v}.reverse] 
  end
end

class CompanyNameCounter
  include NameCounter

  def initialize
    @names = Hash.new(0)
  end

  def field
    'data.netbios_mac_company'
  end
end

class NetbiosNameCounter
  include NameCounter

  def initialize
    @names = Hash.new(0)
  end

  def field
    'data.netbios_hname'
  end
end

company_counter = CompanyNameCounter.new
netbios_counter = NetbiosNameCounter.new

while line=gets
  hash = Oj.load(line.strip)
  company_counter.count(hash)
  netbios_counter.count(hash)
end

summary = {
  'top_companies'     => company_counter.top_names,
  'top_netbios_names' => netbios_counter.top_names
}

puts Oj.dump(summary)
{'fraction_non_alphanumeric': 0.10858257477243173, 'fraction_numerical': 0.005201560468140442, 'mean_word_length': 3.6778115501519757, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7426136', 'n_tokens_mistral': 550, 'n_tokens_neox': 536, 'n_words': 132}