<commit_before>class Ridgepole::Diff
  def initialize(options = {})
    @options = options
  end

  def diff(from, to)
    from = (from || {}).dup
    to = (to || {}).dup

    if @options[:reverse]
      from, to = to, from
    end

    delta = {}
    scan_table_rename(from, to, delta)
    # for reverse option
    scan_table_rename(to, from, delta)

    to.each do |table_name, to_attrs|
      next unless target?(table_name)

      if (from_attrs = from.delete(table_name))
        scan_change(table_name, from_attrs, to_attrs, delta)
      else
        delta[:add] ||= {}
        delta[:add][table_name] = to_attrs
      end
    end

    unless @options[:merge]
      from.each do |table_name, from_attrs|
        next unless target?(table_name)

        delta[:delete] ||= {}
        delta[:delete][table_name] = from_attrs
      end
    end

    Ridgepole::Delta.new(delta, @options)
  end

  private

  def scan_table_rename(from, to, delta, options = {})
    to.dup.each do |table_name, to_attrs|
      next unless target?(table_name)

      if (from_table_name = (to_attrs[:options] || {}).delete(:renamed_from))
        # Already renamed
        next if from[table_name]

        # No existence checking because there is that the table to be read is limited
        #unless from.has_key?(from_table_name)
        #  raise "Table `#{from_table_name}` not found"
        #end

        delta[:rename] ||= {}

        if @options[:reverse]
          delta[:rename][from_table_name] = table_name
        else
          delta[:rename][table_name] = from_table_name
        end

        from.delete(from_table_name)
        to.delete(table_name)
      end
    end
  end

  def scan_change(table_name, from, to, delta)
    from = (from || {}).dup
    to = (to || {}).dup
    table_delta = {}

    scan_options_change(table_name, from[:options], to[:options], table_delta)
    scan_definition_change(from[:definition], to[:definition], from[:indices], table_delta)
    scan_indices_change(from[:indices], to[:indices], to[:definition], table_delta)

    unless table_delta.empty?
      delta[:change] ||= {}
      delta[:change][table_name] = table_delta
    end
  end

  def scan_options_change(table_name, from, to, table_delta)
    unless from == to
      Ridgepole::Logger.instance.warn("[WARNING] Table `#{table_name}` options cannot be changed")
    end
  end

  def scan_definition_change(from, to, from_indices, table_delta)
    from = (from || {}).dup
    to = (to || {}).dup
    definition_delta = {}

    scan_column_rename(from, to, definition_delta)
    # for reverse option
    scan_column_rename(to, from, definition_delta)

    priv_column_name = nil

    to.each do |column_name, to_attrs|
      if (from_attrs = from.delete(column_name))
        normalize_column_options!(from_attrs[:options])
        normalize_column_options!(to_attrs[:options])

        if from_attrs != to_attrs
          definition_delta[:change] ||= {}
          definition_delta[:change][column_name] = to_attrs
        end
      else
        definition_delta[:add] ||= {}
        to_attrs[:options] ||= {}

        unless @options[:merge]
          if priv_column_name
            to_attrs[:options][:after] = priv_column_name
          else
            to_attrs[:options][:first] = true
          end
        end

        definition_delta[:add][column_name] = to_attrs<commit_msg>Fix merge mode<commit_after>class Ridgepole::Diff
  def initialize(options = {})
    @options = options
  end

  def diff(from, to)
    from = (from || {}).dup
    to = (to || {}).dup

    if @options[:reverse]
      from, to = to, from
    end

    delta = {}
    scan_table_rename(from, to, delta)
    # for reverse option
    scan_table_rename(to, from, delta)

    to.each do |table_name, to_attrs|
      next unless target?(table_name)

      if (from_attrs = from.delete(table_name))
        scan_change(table_name, from_attrs, to_attrs, delta)
      else
        delta[:add] ||= {}
        delta[:add][table_name] = to_attrs
      end
    end

    unless @options[:merge]
      from.each do |table_name, from_attrs|
        next unless target?(table_name)

        delta[:delete] ||= {}
        delta[:delete][table_name] = from_attrs
      end
    end

    Ridgepole::Delta.new(delta, @options)
  end

  private

  def scan_table_rename(from, to, delta, options = {})
    to.dup.each do |table_name, to_attrs|
      next unless target?(table_name)

      if (from_table_name = (to_attrs[:options] || {}).delete(:renamed_from))
        # Already renamed
        next if from[table_name]

        # No existence checking because there is that the table to be read is limited
        #unless from.has_key?(from_table_name)
        #  raise "Table `#{from_table_name}` not found"
        #end

        delta[:rename] ||= {}

        if @options[:reverse]
          delta[:rename][from_table_name] = table_name
        else
          delta[:rename][table_name] = from_table_name
        end

        from.delete(from_table_name)
        to.delete(table_name)
      end
    end
  end

  def scan_change(table_name, from, to, delta)
    from = (from || {}).dup
    to = (to || {}).dup
    table_delta = {}

    scan_options_change(table_name, from[:options], to[:options], table_delta)
    scan_definition_change(from[:definition], to[:definition], from[:indices], table_delta)
    scan_indices_change(from[:indices], to[:indices], to[:definition], table_delta)

    unless table_delta.empty?
      delta[:change] ||= {}
      delta[:change][table_name] = table_delta
    end
  end

  def scan_options_change(table_name, from, to, table_delta)
    unless from == to
      Ridgepole::Logger.instance.warn("[WARNING] Table `#{table_name}` options cannot be changed")
    end
  end

  def scan_definition_change(from, to, from_indices, table_delta)
    from = (from || {}).dup
    to = (to || {}).dup
    definition_delta = {}

    scan_column_rename(from, to, definition_delta)
    # for reverse option
    scan_column_rename(to, from, definition_delta)

    priv_column_name = nil

    to.each do |column_name, to_attrs|
      if (from_attrs = from.delete(column_name))
        normalize_column_options!(from_attrs[:options])
        normalize_column_options!(to_attrs[:options])

        if from_attrs != to_attrs
          definition_delta[:change] ||= {}
          definition_delta[:change][column_name] = to_attrs
        end
      else
        definition_delta[:add] ||= {}
        to_attrs[:options] ||= {}

        if priv_column_name
          to_attrs[:options][:after] = priv_column_name
        else
          to_attrs[:options][:first] = true
        end

        definition_delta[:add][column_name] = to_attrs
{'fraction_non_alphanumeric': 0.12094967896072868, 'fraction_numerical': 0.0, 'mean_word_length': 2.7884615384615383, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5485293', 'n_tokens_mistral': 2104, 'n_tokens_neox': 2083, 'n_words': 462}