<commit_before>#!/usr/bin/env ruby

require 'rubygems'
require 'bundler/setup'
require_relative '../lib/kmeans-clusterer'
require_relative './utils/mnist_loader'
require 'narray'
require 'chunky_png'
require 'csv'

train_size = (ARGV[0] || 5000).to_i
runs = (ARGV[1] || 1).to_i

test_size = 200

orig_data, labels = MnistLoader.training_set.get_data_and_labels(train_size + test_size)

# crop 4px border
data = orig_data.map do |row|
  row = NArray.to_na(row)
  row = row.reshape!(28,28)
  row = row[4..23, 4..23]
  row.reshape!(20*20)
end



train_data, train_labels = data.slice(0, train_size), labels.slice(0, train_size)
test_data, test_labels = data.slice(train_size, test_size), labels.slice(train_size, test_size)

puts "Clustering #{train_size} images:"

t = Time.now
kmeans = KMeansClusterer.run(10, train_data, labels: train_labels, runs: runs, scale_data: true, log: true)
elapsed = Time.now - t

# kmeans.clusters.each do |cluster|
#   puts "\n#---\n\n"
#   puts cluster.sorted_points.map(&:label).join(' ')
# end

puts "\nBest of #{runs} runs (total time #{elapsed.round(2)}s):"
puts "10 clusters in #{kmeans.iterations} iterations, #{kmeans.runtime.round(2)}s"
# puts "Silhouette score: #{kmeans.silhouette_score.round(2)}"

puts "\nUsing kmeans to cluster #{test_size} new samples from test set:"

# console output: show lables

predictions_labels = Array.new(10) { [] }

test_data.each.with_index do |row, i|
  label = test_labels[i]
  cluster = kmeans.closest_cluster row
  predictions_labels[cluster.label - 1] << label
end

predictions_labels.each do |vals|
  puts "\n#---\n\n"
  puts vals.join(' ')
end


# png output: show actual images

predictions_images = Array.new(10) { [] }

orig_test = orig_data.slice(train_size, test_size)

test_data.each.with_index do |row, i|
  cluster = kmeans.closest_cluster row
  predictions_images[cluster.label - 1] << orig_test[i]
end

image_size = 28
max_per_row = 25
gridrows, gridcols = 10, 25

@png = ChunkyPNG::Image.new(gridcols * image_size, gridrows * image_size, ChunkyPNG::Color::TRANSPARENT)<commit_msg>no need to scale mnist data<commit_after>#!/usr/bin/env ruby

require 'rubygems'
require 'bundler/setup'
require_relative '../lib/kmeans-clusterer'
require_relative './utils/mnist_loader'
require 'narray'
require 'chunky_png'
require 'csv'

train_size = (ARGV[0] || 5000).to_i
runs = (ARGV[1] || 1).to_i

test_size = 200

orig_data, labels = MnistLoader.training_set.get_data_and_labels(train_size + test_size)

# crop 4px border
data = orig_data.map do |row|
  row = NArray.to_na(row)
  row = row.reshape!(28,28)
  row = row[4..23, 4..23]
  row.reshape!(20*20)
end



train_data, train_labels = data.slice(0, train_size), labels.slice(0, train_size)
test_data, test_labels = data.slice(train_size, test_size), labels.slice(train_size, test_size)

puts "Clustering #{train_size} images:"

t = Time.now
kmeans = KMeansClusterer.run(10, train_data, labels: train_labels, runs: runs, log: true)
elapsed = Time.now - t

# kmeans.clusters.each do |cluster|
#   puts "\n#---\n\n"
#   puts cluster.sorted_points.map(&:label).join(' ')
# end

puts "\nBest of #{runs} runs (total time #{elapsed.round(2)}s):"
puts "10 clusters in #{kmeans.iterations} iterations, #{kmeans.runtime.round(2)}s"
# puts "Silhouette score: #{kmeans.silhouette_score.round(2)}"

puts "\nUsing kmeans to cluster #{test_size} samples from test set:\n\n"

# console output: show lables

predictions_labels = Array.new(10) { [] }

test_data.each.with_index do |row, i|
  label = test_labels[i]
  cluster = kmeans.closest_cluster row
  predictions_labels[cluster.label - 1] << label
end

predictions_labels.each do |vals|
  # puts "\n#---\n\n"
  puts vals.join(' ')
end


# png output: show actual images

predictions_images = Array.new(10) { [] }

orig_test = orig_data.slice(train_size, test_size)

test_data.each.with_index do |row, i|
  cluster = kmeans.closest_cluster row
  predictions_images[cluster.label - 1] << orig_test[i]
end

image_size = 28
max_per_row = 25
gridrows, gridcols = 10, 25

@png = ChunkyPNG::Image.new(gridcols * image_size, gridrows * image_size, ChunkyPNG::Color::TRANSPARENT)
{'fraction_non_alphanumeric': 0.14378927017883036, 'fraction_numerical': 0.02319961333977767, 'mean_word_length': 4.912857142857143, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1409238', 'n_tokens_mistral': 1660, 'n_tokens_neox': 1581, 'n_words': 353}