<commit_before>require 'set'
require 'tsuga/model/point'
require 'tsuga/model/tile'

module Tsuga::Service

  # Aggregates clusters together until no two clusters are closer than
  # a given minimum distance.
  class Aggregator
    # fraction of tile diagonal
    MIN_DISTANCE_RATIO = 0.2

    def initialize(clusters)
      @_clusters = clusters.dup
    end

    def run
      # build the set of pairs (n²/2)
      pairs  = []
      source = _clusters.dup
      while left = source.pop
        source.each do |right| 
          pairs << Pair.new(left, right)
        end
      end

      # pop & merge
      to_delete  = []
      to_persist = Set.new
      while pairs.any?
        best_pair = pairs.min
        break if best_pair.distance > min_distance

        # remove the closest pair
        left, right = best_pair.values

        # remove pairs containing one of the items
        pairs.delete_if { |p| p.has?(left) || p.has?(right) }

        # merge clusters
        left.merge(right)
        _clusters.delete_if { |c| c.id == right.id }
        to_delete << right
        to_persist << left.id

        # create new pairs
        _clusters.each do |cluster|
          next if cluster.id == left.id
          pairs << Pair.new(left, cluster)
        end
      end

      # persistence
      _clusters.each { |c| c.persist! if to_persist.include?(c.id) }
      to_delete.each { |c| c.destroy }
      nil
    end

    # 1/5th of the diagonal of a tile
    def min_distance
      @min_distance ||= begin
        depth = _clusters.first.depth
        point = Tsuga::Model::Point.new.set_coords(0,0)
        tile  = Tsuga::Model::Tile.including(point, :depth => depth)
        (tile.northwest & tile.southeast) * MIN_DISTANCE_RATIO
      end
    end

    private

    attr_reader :_clusters

    # model a pair of clusters such as [a,b] == [b,a]
    # and comparison is based on distance
    class Pair
      include Comparable
      attr_reader :distance

      def initialize(c1, c2)
        @left  = c1
        @right = c2
        @distance = (@left & @right)
      end

      def <=>(other)
        self.distance <=> other.distance
      end

      def ==(other)
        (self.left.id == other.left.id) && (self.right.id == other.right.id)
      end

      def values
        [@left, @right]
      end<commit_msg>Adds sanity check to Pair constructor<commit_after>require 'set'
require 'tsuga/model/point'
require 'tsuga/model/tile'

module Tsuga::Service

  # Aggregates clusters together until no two clusters are closer than
  # a given minimum distance.
  class Aggregator
    # fraction of tile diagonal
    MIN_DISTANCE_RATIO = 0.2

    def initialize(clusters)
      @_clusters = clusters.dup
    end

    def run
      # build the set of pairs (n²/2)
      pairs  = []
      source = _clusters.dup
      while left = source.pop
        source.each do |right| 
          pairs << Pair.new(left, right)
        end
      end

      # pop & merge
      to_delete  = []
      to_persist = Set.new
      while pairs.any?
        best_pair = pairs.min
        break if best_pair.distance > min_distance

        # remove the closest pair
        left, right = best_pair.values

        # remove pairs containing one of the items
        pairs.delete_if { |p| p.has?(left) || p.has?(right) }

        # merge clusters
        left.merge(right)
        _clusters.delete_if { |c| c.id == right.id }
        to_delete << right
        to_persist << left.id

        # create new pairs
        _clusters.each do |cluster|
          next if cluster.id == left.id
          pairs << Pair.new(left, cluster)
        end
      end

      # persistence
      _clusters.each { |c| c.persist! if to_persist.include?(c.id) }
      to_delete.each { |c| c.destroy }
      nil
    end

    # 1/5th of the diagonal of a tile
    def min_distance
      @min_distance ||= begin
        depth = _clusters.first.depth
        point = Tsuga::Model::Point.new.set_coords(0,0)
        tile  = Tsuga::Model::Tile.including(point, :depth => depth)
        (tile.northwest & tile.southeast) * MIN_DISTANCE_RATIO
      end
    end

    private

    attr_reader :_clusters

    # model a pair of clusters such as [a,b] == [b,a]
    # and comparison is based on distance
    class Pair
      include Comparable
      attr_reader :distance

      def initialize(c1, c2)
        raise ArgumentError, 'pair elements must be distinct' if c1.id == c2.id
        @left  = c1
        @right = c2
        @distance = (@left & @right)
      end

      def <=>(other)
        self.distance <=> other.distance
      end

      def ==(other)
        (self.left.id == other.left.id) && (self.right.id == other.right.id)
      end

      def values
        [@left, @right]
      end
{'fraction_non_alphanumeric': 0.0983157894736842, 'fraction_numerical': 0.0050526315789473685, 'mean_word_length': 1.938157081014224, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6864407', 'n_tokens_mistral': 1524, 'n_tokens_neox': 1477, 'n_words': 416}