<commit_before>require 'pp'

class Bicluster
  attr_accessor :left, :right, :vec, :id, :distance
  
  def initialize(options)
    self.vec = options[:vec]
    self.left = options[:left]
    self.right = options[:right]
    self.distance = options[:distance] || 0.0
    self.id = options[:id]
  end
end

module Clusters
  class << self
    
    def run
      row_names, col_names, data = read_blog_data
      # print_cluster(h_cluster(data), row_names)
      pp k_cluster(data)
    end
    
    def k_cluster(rows, k = 4)
      # create a list of min and max values for each point
      ranges = rows[0].size.times.inject([]) do |list, i|
        list << rows.inject([]){|col, row| col << row[i]}.minmax{|a,b| a <=> b}
      end
      # create k random points
      clusters = k.times.inject([]) do |list, j| 
        list << rows[0].size.times.inject([]) do |col, i|
           col << rand * (ranges[i][1] - ranges[i][0]) + ranges[i][0]
        end
      end
      
      last_matches = nil
      best_matches = nil
      100.times do |t|
        puts "Iteration #{t}"
        best_matches = k.times.inject([]){|a, i| a << []}
        
        rows.size.times do |j|
          row = rows[j]
          best_match = 0
          k.times do |i|
            d = pearson(clusters[i], row)
            best_match = i if d < pearson(clusters[best_match], row)
          end
          best_matches[best_match] << j
        end
        
        break if best_matches == last_matches
        last_matches = best_matches
        # move the centroids to the average of their members
        k.times do |i|
          avgs = [0.0] * rows[0].size
          if best_matches[i].size > 0
            for rowid in best_matches[i]
              rows[rowid].size.times do |m|
                avgs[m] += rows[rowid][m]
              end
            end
            avgs.size.times do |j|
              avgs[j] /= best_matches[i].size
            end
            clusters[i] = avgs
          end
        end
      end
      
      return best_matches
    end
    
    def print_cluster(clust, labels = nil, n = 0)
      if clust.id < 0
        puts "-"
      else
        puts labels[clust.id]
      end
      print_cluster(clust.left, labels, n + 1) if clust.left
      print_cluster(clust.right, labels, n + 1) if clust.right
    end
    
    def h_cluster(rows)
      distances = {}
      current_clust_id = -1
      
      clust = rows.size.times.inject([]){|list, c| list << Bicluster.new({:vec => rows[c], :id => c})}
      
      while clust.size > 1
        lowest_pair = [0,1]
        closest = pearson(clust[0].vec, clust[1].vec)
        
        clust.size.times do |i|
          (i + 1).upto(clust.size - 1) do |j|
            if !distances.has_key?([clust[i].id, clust[j].id])
              distances[[clust[i].id, clust[j].id]] = pearson(clust[i].vec, clust[j].vec)
            end
            d = distances[[clust[i].id, clust[j].id]]
            
            if d < closest<commit_msg>added print method for k_cluster output<commit_after>require 'pp'

class Bicluster
  attr_accessor :left, :right, :vec, :id, :distance
  
  def initialize(options)
    self.vec = options[:vec]
    self.left = options[:left]
    self.right = options[:right]
    self.distance = options[:distance] || 0.0
    self.id = options[:id]
  end
end

module Clusters
  class << self
    
    def run
      row_names, col_names, data = read_blog_data
      # print_h_cluster(h_cluster(data), row_names)
      print_k_cluster(k_cluster(data, 10), row_names)
    end
    
    def print_k_cluster(clust, labels)
      puts "------------------------------"
      puts "------------------------------"
      clust.each do |centroid|
        centroid.each do |id|
          puts labels[id]
        end
        puts "------------------------------"
        puts "------------------------------"
      end
    end
    
    def k_cluster(rows, k = 4)
      # create a list of min and max values for each point
      ranges = rows[0].size.times.inject([]) do |list, i|
        list << rows.inject([]){|col, row| col << row[i]}.minmax{|a,b| a <=> b}
      end
      # create k random points
      clusters = k.times.inject([]) do |list, j| 
        list << rows[0].size.times.inject([]) do |col, i|
           col << rand * (ranges[i][1] - ranges[i][0]) + ranges[i][0]
        end
      end
      
      last_matches = nil
      best_matches = nil
      100.times do |t|
        puts "Iteration #{t}"
        best_matches = k.times.inject([]){|a, i| a << []}
        
        rows.size.times do |j|
          row = rows[j]
          best_match = 0
          k.times do |i|
            d = pearson(clusters[i], row)
            best_match = i if d < pearson(clusters[best_match], row)
          end
          best_matches[best_match] << j
        end
        
        break if best_matches == last_matches
        last_matches = best_matches
        # move the centroids to the average of their members
        k.times do |i|
          avgs = [0.0] * rows[0].size
          if best_matches[i].size > 0
            for rowid in best_matches[i]
              rows[rowid].size.times do |m|
                avgs[m] += rows[rowid][m]
              end
            end
            avgs.size.times do |j|
              avgs[j] /= best_matches[i].size
            end
            clusters[i] = avgs
          end
        end
      end
      
      return best_matches
    end
    
    def print_h_cluster(clust, labels = nil, n = 0)
      if clust.id < 0
        puts "-"
      else
        puts labels[clust.id]
      end
      print_cluster(clust.left, labels, n + 1) if clust.left
      print_cluster(clust.right, labels, n + 1) if clust.right
    end
    
    def h_cluster(rows)
      distances = {}
      current_clust_id = -1
      
      clust = rows.size.times.inject([]){|list, c| list << Bicluster.new({:vec => rows[c], :id => c})}
      
      while clust.size > 1
        lowest_pair = [0,1]
        closest = pearson(clust[0].vec, clust[1].vec)
        
        clust.size.times do |i|
          (i + 1).upto(clust.size - 1) do |j|
            if !distances.has_key?([clust[i].id, clust[j].id])
              distances[[clust[i].id, clust[j].id]] = pearson(clust[i].vec, clust[j].vec)
            end
            d = distances[[clust[i].id, clust[j].id]]
            
            if d < closest
{'fraction_non_alphanumeric': 0.1449480642115203, 'fraction_numerical': 0.00912810827824992, 'mean_word_length': 1.619538334707337, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2316999', 'n_tokens_mistral': 2120, 'n_tokens_neox': 2098, 'n_words': 513}