<commit_before>class Ridgepole::Diff
  def initialize(options = {})
    @options = options
    @logger = Ridgepole::Logger.instance
  end

  def diff(from, to, options = {})
    from = (from || {}).deep_dup
    to = (to || {}).deep_dup

    if @options[:reverse]
      from, to = to, from
    end

    delta = {}
    relation_info = {}

    scan_table_rename(from, to, delta)
    # for reverse option
    scan_table_rename(to, from, delta)

    to.each do |table_name, to_attrs|
      collect_relation_info!(table_name, to_attrs, relation_info)

      next unless target?(table_name)

      if (from_attrs = from.delete(table_name))
        @logger.verbose_info("#   #{table_name}")

        unless (attrs_delta = diff_inspect(from_attrs, to_attrs)).empty?
          @logger.verbose_info(attrs_delta)
        end

        scan_change(table_name, from_attrs, to_attrs, delta)
      else
        delta[:add] ||= {}
        delta[:add][table_name] = to_attrs
      end
    end

    scan_relation_info(relation_info)

    unless @options[:merge] or @options[:skip_drop_table]
      from.each do |table_name, from_attrs|
        next unless target?(table_name)

        delta[:delete] ||= {}
        delta[:delete][table_name] = from_attrs
      end
    end

    delta[:execute] = options[:execute]

    Ridgepole::Delta.new(delta, @options)
  end

  private

  def scan_table_rename(from, to, delta, options = {})
    to.dup.each do |table_name, to_attrs|
      next unless target?(table_name)

      if (from_table_name = (to_attrs[:options] || {}).delete(:renamed_from))
        from_table_name = from_table_name.to_s if from_table_name

        # Already renamed
        next if from[table_name]

        # No existence checking because there is that the table to be read is limited
        #unless from.has_key?(from_table_name)
        #  raise "Table `#{from_table_name}` not found"
        #end

        delta[:rename] ||= {}

        if @options[:reverse]
          delta[:rename][from_table_name] = table_name
        else
          delta[:rename][table_name] = from_table_name
        end

        from.delete(from_table_name)
        to.delete(table_name)
      end
    end
  end

  def scan_change(table_name, from, to, delta)
    from = (from || {}).dup
    to = (to || {}).dup
    table_delta = {}

    scan_options_change(table_name, from[:options], to[:options], table_delta)
    scan_definition_change(from[:definition], to[:definition], from[:indices], table_name, from[:options], table_delta)
    scan_indices_change(from[:indices], to[:indices], to[:definition], table_delta, from[:options], to[:options])
    scan_foreign_keys_change(from[:foreign_keys], to[:foreign_keys], table_delta, @options)

    unless table_delta.empty?
      delta[:change] ||= {}
      delta[:change][table_name] = table_delta
    end
  end

  def scan_options_change(table_name, from, to, table_delta)
    from = (from || {}).dup
    to = (to || {}).dup

    normalize_default_proc_options!(from, to)

    from_options = from[:options] || {}
    to_options = to[:options] || {}

    if @options[:ignore_table_comment]
      from.delete(:comment)
      to.delete(:comment)
    end

    if @options[:mysql_change_table_options] and from_options != to_options and Ridgepole::ConnectionAdapters.mysql?
      from.delete(:options)
      to.delete(:options)
      table_delta[:table_options] = to_options
    end

    unless from == to
      @logger.warn(<<-EOS)
[WARNING] No difference of schema configuration for table `#{table_name}` but table options differ.
  from: #{from}
    to: #{to}
      EOS
    end
  end

  def scan_definition_change(from, to, from_indices, table_name, table_options, table_delta)
    from = (from || {}).dup
    to = (to || {}).dup
    definition_delta = {}

    scan_column_rename(from, to, definition_delta)
    # for reverse option
    scan_column_rename(to, from, definition_delta)

    if table_options[:id] == false or table_options[:primary_key].is_a?(Array)
      priv_column_name = nil
    else
      priv_column_name = table_options[:primary_key] || 'id'
    end

    to.each do |column_name, to_attrs|
      if (from_attrs = from.delete(column_name))
        normalize_column_options!(from_attrs)
        normalize_column_options!(to_attrs)

        unless compare_column_attrs(from_attrs, to_attrs)
          definition_delta[:change] ||= {}
          to_attrs = fix_change_column_options(table_name, from_attrs, to_attrs)
          definition_delta[:change][column_name] = to_attrs
        end
      else
        definition_delta[:add] ||= {}
        to_attrs[:options] ||= {}

        if priv_column_name
          to_attrs[:options][:after] = priv_column_name
        else
          to_attrs[:options][:first] = true
        end

        definition_delta[:add][column_name] = to_attrs
      end

      priv_column_name = column_name
    end

    if Ridgepole::ConnectionAdapters.postgresql?
      added_size = 0
      to.reverse_each.with_index do |(column_name, to_attrs), i|
        if to_attrs[:options].delete(:after)
          if added_size != i
            @logger.warn("[WARNING] PostgreSQL doesn't support adding a new column except for the last position. #{table_name}.#{column_name} will be added to the last.")
          end
          added_size += 1
        end
      end
    end

    unless @options[:merge]
      from.each do |column_name, from_attrs|
        definition_delta[:delete] ||= {}
        definition_delta[:delete][column_name] = from_attrs

        if from_indices
          modified_indices = []

          from_indices.each do |name, attrs|
            if attrs[:column_name].is_a?(Array) && attrs[:column_name].delete(column_name)
              modified_indices << name
            end
          end

          # In PostgreSQL, the index is deleted when the column is deleted
          if @options[:index_removed_drop_column]
            from_indices.reject! do |name, attrs|
              modified_indices.include?(name)
            end
          end

          from_indices.reject! do |name, attrs|
            attrs[:column_name].is_a?(Array) && attrs[:column_name].empty?
          end
        end
      end
    end

    unless definition_delta.empty?
      table_delta[:definition] = definition_delta
    end
  end

  def scan_column_rename(from, to, definition_delta)
    to.dup.each do |column_name, to_attrs|
      if (from_column_name = (to_attrs[:options] || {}).delete(:renamed_from))
        from_column_name = from_column_name.to_s if from_column_name

        # Already renamed
        next if from[column_name]

        unless from.has_key?(from_column_name)
          raise "Column `#{from_column_name}` not found"
        end

        definition_delta[:rename] ||= {}

        if @options[:reverse]
          definition_delta[:rename][from_column_name] = column_name
        else
          definition_delta[:rename][column_name] = from_column_name
        end

        from.delete(from_column_name)
        to.delete(column_name)
      end
    end
  end

  def scan_indices_change(from, to, to_columns, table_delta, from_table_options, to_table_options)
    from = (from || {}).dup
    to = (to || {}).dup
    indices_delta = {}

    to.each do |index_name, to_attrs|
      if index_name.kind_of?(Array)
        from_index_name, from_attrs = from.find {|name, attrs| attrs[:column_name] == index_name }

        if from_attrs
          from.delete(from_index_name)
          from_attrs[:options].delete(:name)
        end
      else
        from_attrs = from.delete(index_name)
      end

      if from_attrs
        normalize_index_options!(from_attrs[:options])
        normalize_index_options!(to_attrs[:options])

        if from_attrs != to_attrs
          indices_delta[:add] ||= {}
          indices_delta[:add][index_name] = to_attrs

          unless @options[:merge]
            if columns_all_include?(from_attrs[:column_name], to_columns.keys, to_table_options)
              indices_delta[:delete] ||= {}
              indices_delta[:delete][index_name] = from_attrs
            end
          end
        end
      else
        indices_delta[:add] ||= {}
        indices_delta[:add][index_name] = to_attrs
      end
    end

    unless @options[:merge]
      from.each do |index_name, from_attrs|
        if columns_all_include?(from_attrs[:column_name], to_columns.keys, to_table_options)
          indices_delta[:delete] ||= {}
          indices_delta[:delete][index_name] = from_attrs
        end
      end
    end

    unless indices_delta.empty?
      table_delta[:indices] = indices_delta
    end
  end

  def target?(table_name)
    if @options[:tables] and @options[:tables].include?(table_name)
      true
    elsif @options[:ignore_tables] and @options[:ignore_tables].any? {|i| i =~ table_name }
      false
    elsif @options[:tables]
      false
    else
      true
    end
  end

  def normalize_column_options!(attrs)
    opts = attrs[:options]
    opts[:null] = true unless opts.has_key?(:null)
    default_limit = Ridgepole::DefaultsLimit.default_limit(attrs[:type], @options)
    opts.delete(:limit) if opts[:limit] == default_limit

    # XXX: MySQL only?
    if not opts.has_key?(:default)
      opts[:default] = nil
    end

    if Ridgepole::ConnectionAdapters.mysql?
      opts[:unsigned] = false unless opts.has_key?(:unsigned)

      if attrs[:type] == :integer and opts[:limit] == Ridgepole::DefaultsLimit.default_limit(:bigint, @options)
        attrs[:type] = :bigint
        opts.delete(:limit)
      end
    end
  end

  def normalize_index_options!(opts)
    # XXX: MySQL only?
    opts[:using] = :btree unless opts.has_key?(:using)
    opts[:unique] = false unless opts.has_key?(:unique)
  end

  def columns_all_include?(expected_columns, actual_columns, table_options)
    unless expected_columns.is_a?(Array)
      return true
    end

    if table_options[:id] != false and not table_options[:primary_key].is_a?(Array)
      actual_columns = actual_columns + [(table_options[:primary_key] || 'id').to_s]
    end

    expected_columns.all? {|i| actual_columns.include?(i) }
  end

  def scan_foreign_keys_change(from, to, table_delta, options)
    from = (from || {}).dup
    to = (to || {}).dup
    foreign_keys_delta = {}

    to.each do |foreign_key_name_or_tables, to_attrs|
      from_attrs = from.delete(foreign_key_name_or_tables)

      if from_attrs
        if from_attrs != to_attrs
          foreign_keys_delta[:add] ||= {}
          foreign_keys_delta[:add][foreign_key_name_or_tables] = to_attrs

          unless options[:merge]
            foreign_keys_delta[:delete] ||= {}
            foreign_keys_delta[:delete][foreign_key_name_or_tables] = from_attrs
          end
        end
      else
        foreign_keys_delta[:add] ||= {}
        foreign_keys_delta[:add][foreign_key_name_or_tables] = to_attrs
      end
    end

    unless options[:merge]
      from.each do |foreign_key_name_or_tables, from_attrs|
        foreign_keys_delta[:delete] ||= {}
        foreign_keys_delta[:delete][foreign_key_name_or_tables] = from_attrs
      end
    end

    unless foreign_keys_delta.empty?
      table_delta[:foreign_keys] = foreign_keys_delta
    end
  end

  # XXX: MySQL only?
  # https://github.com/rails/rails/blob/v4.2.1/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb#L760
  # https://github.com/rails/rails/blob/v4.2.1/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb#L102
  def fix_change_column_options(table_name, from_attrs, to_attrs)
    # default: 0, null: false -> default: nil, null: false | default: nil
    # default: 0, null: false ->               null: false | default: nil
    # default: 0, null: false -> default: nil, null: true  | default: nil, null: true
    # default: 0, null: false ->               null: true  | default: nil, null: true
    # default: 0, null: true  -> default: nil, null: true  | default: nil
    # default: 0, null: true  ->               null: true  | default: nil
    # default: 0, null: true  -> default: nil, null: false | default: nil, null: false (`default: nil` is ignored)
    # default: 0, null: true ->                null: false | default: nil, null: false (`default: nil` is ignored)

    if from_attrs[:options][:default] != to_attrs[:options][:default] and from_attrs[:options][:null] == to_attrs[:options][:null]
      to_attrs = to_attrs.deep_dup
      to_attrs[:options].delete(:null)
    end

    if Ridgepole::ConnectionAdapters.mysql? and ActiveRecord::VERSION::STRING =~ /\A5\.0\./
      if to_attrs[:options][:default] == nil and to_attrs[:options][:null] == false
        Ridgepole::Logger.instance.warn("[WARNING] Table `#{table_name}`: `default: nil` is ignored when `null: false`. Please apply twice")
      end
    end

    to_attrs
  end

  def compare_column_attrs(attrs1, attrs2)
    attrs1 = attrs1.merge(:options => attrs1.fetch(:options, {}).dup)
    attrs2 = attrs2.merge(:options => attrs2.fetch(:options, {}).dup)
    normalize_default_proc_options!(attrs1[:options], attrs2[:options])

    if @options[:skip_column_comment_change]
      attrs1.fetch(:options).delete(:comment)
      attrs2.fetch(:options).delete(:comment)
    end

    attrs1 == attrs2
  end

  def normalize_default_proc_options!(opts1, opts2)
    if opts1[:default].kind_of?(Proc) and opts2[:default].kind_of?(Proc)
      opts1[:default] = opts1[:default].call
      opts2[:default] = opts2[:default].call
    end
  end

  def diff_inspect(obj1, obj2, options = {})
    obj1 = Ridgepole::Ext::PpSortHash.extend_if_hash(obj1)
    obj2 = Ridgepole::Ext::PpSortHash.extend_if_hash(obj2)

    diffy = Diffy::Diff.new(
      obj1.pretty_inspect,
      obj2.pretty_inspect,
      :diff => '-u'
    )

    diffy.to_s(@options[:color] ? :color : :text).gsub(/\s+\z/m, '')
  end

  def collect_relation_info!(table_name, table_attr, relation_info)
    return unless @options[:check_relation_type]

    attrs_by_column = {}
    definition = table_attr[:definition] || {}

    definition.each do |column_name, column_attrs|
      if column_name =~ /\w+_id\z/
        attrs_by_column[column_name] = column_attrs.dup
      end
    end

    relation_info[table_name] = {
      :options => table_attr[:options] || {},
      :columns => attrs_by_column,
    }
  end

  def scan_relation_info(relation_info)
    return unless @options[:check_relation_type]

    relation_info.each do |child_table, table_info|
      next unless target?(child_table)

      attrs_by_column = table_info.fetch(:columns)
      parent_table_info = nil

      attrs_by_column.each do |column_name, column_attrs|
        parent_table = column_name.sub(/_id\z/, '')

        [parent_table.pluralize, parent_table.singularize].each do |table_name|
          parent_table_info = relation_info[table_name]

          if parent_table_info
            parent_table = table_name
            break
          end
        end

        next unless parent_table_info

        table_options = parent_table_info.fetch(:options)

        parent_column_info = {
          :type => table_options[:id] || @options[:check_relation_type].to_sym,
          :unsigned => table_options[:unsigned],
        }

        child_column_info = {
          :type => column_attrs[:type],
          :unsigned => column_attrs.fetch(:options, {})[:unsigned],
        }

        [parent_column_info, child_column_info].each do |column_info|
          unless column_info[:unsigned]
            column_info.delete(:unsigned)
          end

          # for PostgreSQL
          column_info[:type] = {
            :serial => :integer,
            :bigserial => :bigint,
          }.fetch(column_info[:type], column_info[:type])
        end

        if parent_column_info != child_column_info
          parent_label = "#{parent_table}.id"
          child_label = "#{child_table}.#{column_name}"
          label_len = [parent_label.length, child_label.length].max
          parent_column_info_str = parent_column_info.inspect.slice(1...-1)
          child_column_info_str = child_column_info.inspect.slice(1...-1)

          @logger.warn(<<-EOS % [label_len, parent_label, label_len, child_label])
[WARNING] Relation column type is different.
  %*s: #{parent_column_info_str}
  %*s: #{child_column_info_str}
          EOS
        end
      end
    end
  end
end
<commit_msg>Fix for create_table default: nil<commit_after>class Ridgepole::Diff
  def initialize(options = {})
    @options = options
    @logger = Ridgepole::Logger.instance
  end

  def diff(from, to, options = {})
    from = (from || {}).deep_dup
    to = (to || {}).deep_dup

    if @options[:reverse]
      from, to = to, from
    end

    delta = {}
    relation_info = {}

    scan_table_rename(from, to, delta)
    # for reverse option
    scan_table_rename(to, from, delta)

    to.each do |table_name, to_attrs|
      collect_relation_info!(table_name, to_attrs, relation_info)

      next unless target?(table_name)

      if (from_attrs = from.delete(table_name))
        @logger.verbose_info("#   #{table_name}")

        unless (attrs_delta = diff_inspect(from_attrs, to_attrs)).empty?
          @logger.verbose_info(attrs_delta)
        end

        scan_change(table_name, from_attrs, to_attrs, delta)
      else
        delta[:add] ||= {}
        delta[:add][table_name] = to_attrs
      end
    end

    scan_relation_info(relation_info)

    unless @options[:merge] or @options[:skip_drop_table]
      from.each do |table_name, from_attrs|
        next unless target?(table_name)

        delta[:delete] ||= {}
        delta[:delete][table_name] = from_attrs
      end
    end

    delta[:execute] = options[:execute]

    Ridgepole::Delta.new(delta, @options)
  end

  private

  def scan_table_rename(from, to, delta, options = {})
    to.dup.each do |table_name, to_attrs|
      next unless target?(table_name)

      if (from_table_name = (to_attrs[:options] || {}).delete(:renamed_from))
        from_table_name = from_table_name.to_s if from_table_name

        # Already renamed
        next if from[table_name]

        # No existence checking because there is that the table to be read is limited
        #unless from.has_key?(from_table_name)
        #  raise "Table `#{from_table_name}` not found"
        #end

        delta[:rename] ||= {}

        if @options[:reverse]
          delta[:rename][from_table_name] = table_name
        else
          delta[:rename][table_name] = from_table_name
        end

        from.delete(from_table_name)
        to.delete(table_name)
      end
    end
  end

  def scan_change(table_name, from, to, delta)
    from = (from || {}).dup
    to = (to || {}).dup
    table_delta = {}

    scan_options_change(table_name, from[:options], to[:options], table_delta)
    scan_definition_change(from[:definition], to[:definition], from[:indices], table_name, from[:options], table_delta)
    scan_indices_change(from[:indices], to[:indices], to[:definition], table_delta, from[:options], to[:options])
    scan_foreign_keys_change(from[:foreign_keys], to[:foreign_keys], table_delta, @options)

    unless table_delta.empty?
      delta[:change] ||= {}
      delta[:change][table_name] = table_delta
    end
  end

  def scan_options_change(table_name, from, to, table_delta)
    from = (from || {}).dup
    to = (to || {}).dup

    normalize_default_proc_options!(from, to)

    from_options = from[:options] || {}
    to_options = to[:options] || {}

    if @options[:ignore_table_comment]
      from.delete(:comment)
      to.delete(:comment)
    end

    [from, to].each do |table_attrs|
      if table_attrs.has_key?(:default) and table_attrs[:default].nil?
        table_attrs.delete(:default)
      end
    end

    if @options[:mysql_change_table_options] and from_options != to_options and Ridgepole::ConnectionAdapters.mysql?
      from.delete(:options)
      to.delete(:options)
      table_delta[:table_options] = to_options
    end

    unless from == to
      @logger.warn(<<-EOS)
[WARNING] No difference of schema configuration for table `#{table_name}` but table options differ.
  from: #{from}
    to: #{to}
      EOS
    end
  end

  def scan_definition_change(from, to, from_indices, table_name, table_options, table_delta)
    from = (from || {}).dup
    to = (to || {}).dup
    definition_delta = {}

    scan_column_rename(from, to, definition_delta)
    # for reverse option
    scan_column_rename(to, from, definition_delta)

    if table_options[:id] == false or table_options[:primary_key].is_a?(Array)
      priv_column_name = nil
    else
      priv_column_name = table_options[:primary_key] || 'id'
    end

    to.each do |column_name, to_attrs|
      if (from_attrs = from.delete(column_name))
        normalize_column_options!(from_attrs)
        normalize_column_options!(to_attrs)

        unless compare_column_attrs(from_attrs, to_attrs)
          definition_delta[:change] ||= {}
          to_attrs = fix_change_column_options(table_name, from_attrs, to_attrs)
          definition_delta[:change][column_name] = to_attrs
        end
      else
        definition_delta[:add] ||= {}
        to_attrs[:options] ||= {}

        if priv_column_name
          to_attrs[:options][:after] = priv_column_name
        else
          to_attrs[:options][:first] = true
        end

        definition_delta[:add][column_name] = to_attrs
      end

      priv_column_name = column_name
    end

    if Ridgepole::ConnectionAdapters.postgresql?
      added_size = 0
      to.reverse_each.with_index do |(column_name, to_attrs), i|
        if to_attrs[:options].delete(:after)
          if added_size != i
            @logger.warn("[WARNING] PostgreSQL doesn't support adding a new column except for the last position. #{table_name}.#{column_name} will be added to the last.")
          end
          added_size += 1
        end
      end
    end

    unless @options[:merge]
      from.each do |column_name, from_attrs|
        definition_delta[:delete] ||= {}
        definition_delta[:delete][column_name] = from_attrs

        if from_indices
          modified_indices = []

          from_indices.each do |name, attrs|
            if attrs[:column_name].is_a?(Array) && attrs[:column_name].delete(column_name)
              modified_indices << name
            end
          end

          # In PostgreSQL, the index is deleted when the column is deleted
          if @options[:index_removed_drop_column]
            from_indices.reject! do |name, attrs|
              modified_indices.include?(name)
            end
          end

          from_indices.reject! do |name, attrs|
            attrs[:column_name].is_a?(Array) && attrs[:column_name].empty?
          end
        end
      end
    end

    unless definition_delta.empty?
      table_delta[:definition] = definition_delta
    end
  end

  def scan_column_rename(from, to, definition_delta)
    to.dup.each do |column_name, to_attrs|
      if (from_column_name = (to_attrs[:options] || {}).delete(:renamed_from))
        from_column_name = from_column_name.to_s if from_column_name

        # Already renamed
        next if from[column_name]

        unless from.has_key?(from_column_name)
          raise "Column `#{from_column_name}` not found"
        end

        definition_delta[:rename] ||= {}

        if @options[:reverse]
          definition_delta[:rename][from_column_name] = column_name
        else
          definition_delta[:rename][column_name] = from_column_name
        end

        from.delete(from_column_name)
        to.delete(column_name)
      end
    end
  end

  def scan_indices_change(from, to, to_columns, table_delta, from_table_options, to_table_options)
    from = (from || {}).dup
    to = (to || {}).dup
    indices_delta = {}

    to.each do |index_name, to_attrs|
      if index_name.kind_of?(Array)
        from_index_name, from_attrs = from.find {|name, attrs| attrs[:column_name] == index_name }

        if from_attrs
          from.delete(from_index_name)
          from_attrs[:options].delete(:name)
        end
      else
        from_attrs = from.delete(index_name)
      end

      if from_attrs
        normalize_index_options!(from_attrs[:options])
        normalize_index_options!(to_attrs[:options])

        if from_attrs != to_attrs
          indices_delta[:add] ||= {}
          indices_delta[:add][index_name] = to_attrs

          unless @options[:merge]
            if columns_all_include?(from_attrs[:column_name], to_columns.keys, to_table_options)
              indices_delta[:delete] ||= {}
              indices_delta[:delete][index_name] = from_attrs
            end
          end
        end
      else
        indices_delta[:add] ||= {}
        indices_delta[:add][index_name] = to_attrs
      end
    end

    unless @options[:merge]
      from.each do |index_name, from_attrs|
        if columns_all_include?(from_attrs[:column_name], to_columns.keys, to_table_options)
          indices_delta[:delete] ||= {}
          indices_delta[:delete][index_name] = from_attrs
        end
      end
    end

    unless indices_delta.empty?
      table_delta[:indices] = indices_delta
    end
  end

  def target?(table_name)
    if @options[:tables] and @options[:tables].include?(table_name)
      true
    elsif @options[:ignore_tables] and @options[:ignore_tables].any? {|i| i =~ table_name }
      false
    elsif @options[:tables]
      false
    else
      true
    end
  end

  def normalize_column_options!(attrs)
    opts = attrs[:options]
    opts[:null] = true unless opts.has_key?(:null)
    default_limit = Ridgepole::DefaultsLimit.default_limit(attrs[:type], @options)
    opts.delete(:limit) if opts[:limit] == default_limit

    # XXX: MySQL only?
    if not opts.has_key?(:default)
      opts[:default] = nil
    end

    if Ridgepole::ConnectionAdapters.mysql?
      opts[:unsigned] = false unless opts.has_key?(:unsigned)

      if attrs[:type] == :integer and opts[:limit] == Ridgepole::DefaultsLimit.default_limit(:bigint, @options)
        attrs[:type] = :bigint
        opts.delete(:limit)
      end
    end
  end

  def normalize_index_options!(opts)
    # XXX: MySQL only?
    opts[:using] = :btree unless opts.has_key?(:using)
    opts[:unique] = false unless opts.has_key?(:unique)
  end

  def columns_all_include?(expected_columns, actual_columns, table_options)
    unless expected_columns.is_a?(Array)
      return true
    end

    if table_options[:id] != false and not table_options[:primary_key].is_a?(Array)
      actual_columns = actual_columns + [(table_options[:primary_key] || 'id').to_s]
    end

    expected_columns.all? {|i| actual_columns.include?(i) }
  end

  def scan_foreign_keys_change(from, to, table_delta, options)
    from = (from || {}).dup
    to = (to || {}).dup
    foreign_keys_delta = {}

    to.each do |foreign_key_name_or_tables, to_attrs|
      from_attrs = from.delete(foreign_key_name_or_tables)

      if from_attrs
        if from_attrs != to_attrs
          foreign_keys_delta[:add] ||= {}
          foreign_keys_delta[:add][foreign_key_name_or_tables] = to_attrs

          unless options[:merge]
            foreign_keys_delta[:delete] ||= {}
            foreign_keys_delta[:delete][foreign_key_name_or_tables] = from_attrs
          end
        end
      else
        foreign_keys_delta[:add] ||= {}
        foreign_keys_delta[:add][foreign_key_name_or_tables] = to_attrs
      end
    end

    unless options[:merge]
      from.each do |foreign_key_name_or_tables, from_attrs|
        foreign_keys_delta[:delete] ||= {}
        foreign_keys_delta[:delete][foreign_key_name_or_tables] = from_attrs
      end
    end

    unless foreign_keys_delta.empty?
      table_delta[:foreign_keys] = foreign_keys_delta
    end
  end

  # XXX: MySQL only?
  # https://github.com/rails/rails/blob/v4.2.1/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb#L760
  # https://github.com/rails/rails/blob/v4.2.1/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb#L102
  def fix_change_column_options(table_name, from_attrs, to_attrs)
    # default: 0, null: false -> default: nil, null: false | default: nil
    # default: 0, null: false ->               null: false | default: nil
    # default: 0, null: false -> default: nil, null: true  | default: nil, null: true
    # default: 0, null: false ->               null: true  | default: nil, null: true
    # default: 0, null: true  -> default: nil, null: true  | default: nil
    # default: 0, null: true  ->               null: true  | default: nil
    # default: 0, null: true  -> default: nil, null: false | default: nil, null: false (`default: nil` is ignored)
    # default: 0, null: true ->                null: false | default: nil, null: false (`default: nil` is ignored)

    if from_attrs[:options][:default] != to_attrs[:options][:default] and from_attrs[:options][:null] == to_attrs[:options][:null]
      to_attrs = to_attrs.deep_dup
      to_attrs[:options].delete(:null)
    end

    if Ridgepole::ConnectionAdapters.mysql? and ActiveRecord::VERSION::STRING =~ /\A5\.0\./
      if to_attrs[:options][:default] == nil and to_attrs[:options][:null] == false
        Ridgepole::Logger.instance.warn("[WARNING] Table `#{table_name}`: `default: nil` is ignored when `null: false`. Please apply twice")
      end
    end

    to_attrs
  end

  def compare_column_attrs(attrs1, attrs2)
    attrs1 = attrs1.merge(:options => attrs1.fetch(:options, {}).dup)
    attrs2 = attrs2.merge(:options => attrs2.fetch(:options, {}).dup)
    normalize_default_proc_options!(attrs1[:options], attrs2[:options])

    if @options[:skip_column_comment_change]
      attrs1.fetch(:options).delete(:comment)
      attrs2.fetch(:options).delete(:comment)
    end

    attrs1 == attrs2
  end

  def normalize_default_proc_options!(opts1, opts2)
    if opts1[:default].kind_of?(Proc) and opts2[:default].kind_of?(Proc)
      opts1[:default] = opts1[:default].call
      opts2[:default] = opts2[:default].call
    end
  end

  def diff_inspect(obj1, obj2, options = {})
    obj1 = Ridgepole::Ext::PpSortHash.extend_if_hash(obj1)
    obj2 = Ridgepole::Ext::PpSortHash.extend_if_hash(obj2)

    diffy = Diffy::Diff.new(
      obj1.pretty_inspect,
      obj2.pretty_inspect,
      :diff => '-u'
    )

    diffy.to_s(@options[:color] ? :color : :text).gsub(/\s+\z/m, '')
  end

  def collect_relation_info!(table_name, table_attr, relation_info)
    return unless @options[:check_relation_type]

    attrs_by_column = {}
    definition = table_attr[:definition] || {}

    definition.each do |column_name, column_attrs|
      if column_name =~ /\w+_id\z/
        attrs_by_column[column_name] = column_attrs.dup
      end
    end

    relation_info[table_name] = {
      :options => table_attr[:options] || {},
      :columns => attrs_by_column,
    }
  end

  def scan_relation_info(relation_info)
    return unless @options[:check_relation_type]

    relation_info.each do |child_table, table_info|
      next unless target?(child_table)

      attrs_by_column = table_info.fetch(:columns)
      parent_table_info = nil

      attrs_by_column.each do |column_name, column_attrs|
        parent_table = column_name.sub(/_id\z/, '')

        [parent_table.pluralize, parent_table.singularize].each do |table_name|
          parent_table_info = relation_info[table_name]

          if parent_table_info
            parent_table = table_name
            break
          end
        end

        next unless parent_table_info

        table_options = parent_table_info.fetch(:options)

        parent_column_info = {
          :type => table_options[:id] || @options[:check_relation_type].to_sym,
          :unsigned => table_options[:unsigned],
        }

        child_column_info = {
          :type => column_attrs[:type],
          :unsigned => column_attrs.fetch(:options, {})[:unsigned],
        }

        [parent_column_info, child_column_info].each do |column_info|
          unless column_info[:unsigned]
            column_info.delete(:unsigned)
          end

          # for PostgreSQL
          column_info[:type] = {
            :serial => :integer,
            :bigserial => :bigint,
          }.fetch(column_info[:type], column_info[:type])
        end

        if parent_column_info != child_column_info
          parent_label = "#{parent_table}.id"
          child_label = "#{child_table}.#{column_name}"
          label_len = [parent_label.length, child_label.length].max
          parent_column_info_str = parent_column_info.inspect.slice(1...-1)
          child_column_info_str = child_column_info.inspect.slice(1...-1)

          @logger.warn(<<-EOS % [label_len, parent_label, label_len, child_label])
[WARNING] Relation column type is different.
  %*s: #{parent_column_info_str}
  %*s: #{child_column_info_str}
          EOS
        end
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.11676501003710688, 'fraction_numerical': 0.003528195145690127, 'mean_word_length': 2.865843621399177, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 41, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3179701', 'n_tokens_mistral': 10424, 'n_tokens_neox': 10206, 'n_words': 2069}