<commit_before># encoding: UTF-8

module Spontaneous
  class Change < Sequel::Model(:changes)
    class << self
      alias_method :sequel_plugin, :plugin
    end

    sequel_plugin :serialization, :yajl, :modified_list
    sequel_plugin :timestamps

    @@instance = nil

    class << self
      def record
        entry_point = @@instance.nil?
        @@instance ||= self.new(:modified_list => [])
        yield if block_given?
      ensure
        if entry_point and !@@instance.modified_list.empty?
          @@instance.save
        end
        @@instance = nil if entry_point
      end

      def recording?
        !@@instance.nil?
      end

      def push(page)
        if @@instance
          @@instance.push(page)
        end
      end

      def outstanding
        dependencies = dependency_map
        dependencies.map { |d| Spontaneous::Collections::ChangeSet.new(d) }
      end

      def dependency_map
        grouped_changes = self.valid.map { |c| [c] }
        begin
          modified = false
          grouped_changes.each_with_index do |inner, i|
            inner_ids = inner.map { |c| c.modified_list }.flatten
            grouped_changes[(i+1)..-1].each_with_index do |outer, j|
              outer_ids = outer.map { |c| c.modified_list }.flatten
              unless (inner_ids & outer_ids).empty?
                modified = true
                grouped_changes.delete(outer)
                grouped_changes[i] += outer
              end
            end
          end
        end while modified

        grouped_changes
      end

      def valid
        invalid, valid = self.all.partition { |change| change.modified.empty? }
        self.filter(:id => invalid.map { |change| change.id }).delete
        valid
      end

      def export
        outstanding.map { |change_set| change_set.export }
      end

      def serialise_http(user = nil)
        Spontaneous.serialise_http(export)
      end
    end


    def after_initialize
      super
      self.modified_list ||= []
    end

    def push(page)
      self.modified_list << page.id
    end

    def before_update
      self.modified_list.uniq!
    end

    def modified
      @modified ||= modified_list.map { |id| Content[id] }.compact
    end

    alias_method :pages, :modified

    def &(change)
      self.modified_list & change.modified_list
    end

    def export
      {
        :id => self.id,
        :created_at => self.created_at.to_s,
        :page_ids => modified_list
      }
    end
  end
end
<commit_msg>Reject duplicate change set members<commit_after># encoding: UTF-8

module Spontaneous
  class Change < Sequel::Model(:changes)
    class << self
      alias_method :sequel_plugin, :plugin
    end

    sequel_plugin :serialization, :yajl, :modified_list
    sequel_plugin :timestamps

    @@instance = nil

    class << self
      def record
        entry_point = @@instance.nil?
        @@instance ||= self.new(:modified_list => [])
        yield if block_given?
      ensure
        if entry_point and !@@instance.modified_list.empty?
          @@instance.save
        end
        @@instance = nil if entry_point
      end

      def recording?
        !@@instance.nil?
      end

      def push(page)
        if @@instance
          @@instance.push(page)
        end
      end

      def outstanding
        dependencies = dependency_map
        dependencies.map { |d| Spontaneous::Collections::ChangeSet.new(d) }
      end

      def dependency_map
        grouped_changes = self.valid.map { |c| [c] }
        begin
          modified = false
          grouped_changes.each_with_index do |inner, i|
            inner_ids = inner.map { |c| c.modified_list }.flatten
            grouped_changes[(i+1)..-1].each_with_index do |outer, j|
              outer_ids = outer.map { |c| c.modified_list }.flatten
              unless (inner_ids & outer_ids).empty?
                modified = true
                grouped_changes.delete(outer)
                grouped_changes[i] += outer
              end
            end
          end
        end while modified

        grouped_changes
      end

      def valid
        invalid, valid = self.all.partition { |change| change.modified.empty? }
        self.filter(:id => invalid.map { |change| change.id }).delete
        valid
      end

      def export
        outstanding.map { |change_set| change_set.export }
      end

      def serialise_http(user = nil)
        Spontaneous.serialise_http(export)
      end
    end


    def after_initialize
      super
      self.modified_list ||= []
    end

    def push(page)
      self.modified_list << page.id unless self.modified_list.include?(page.id)
    end

    def before_update
      self.modified_list.uniq!
    end

    def modified
      @modified ||= modified_list.map { |id| Content[id] }.compact
    end

    alias_method :pages, :modified

    def &(change)
      self.modified_list & change.modified_list
    end

    def export
      {
        :id => self.id,
        :created_at => self.created_at.to_s,
        :page_ids => modified_list
      }
    end
  end
end

{'fraction_non_alphanumeric': 0.09169618260527351, 'fraction_numerical': 0.0011806375442739079, 'mean_word_length': 1.858830146231721, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2100728', 'n_tokens_mistral': 1535, 'n_tokens_neox': 1473, 'n_words': 354}