<commit_before>require_relative "object_table/version"
require_relative "object_table/basic_grid"
require_relative "object_table/table_methods"
require_relative "object_table/view"
require_relative "object_table/static_view"
require_relative "object_table/column"
require_relative "object_table/grouped"
require_relative "object_table/temp_grouped"

class ObjectTable
  include TableMethods

  attr_reader :columns

  def initialize(columns = {})
    super()

    unless columns.is_a? BasicGrid
      columns = BasicGrid[columns]
    end
    columns._ensure_uniform_columns!
    @columns = columns

    @columns.each do |k, v|
      @columns[k] = ObjectTable::Column.make(v)
    end
  end

  def add_column(name, typecode='object', *args)
    col = ObjectTable::Column.new(typecode, *args, nrows)
    columns[name] = col
  end

  def stack!(*others)
    new_values = Hash[colnames.zip(ncols.times.map{[]})]

    others.each do |x|
      case x
      when ObjectTable::TableMethods
        x = x.columns
      when ObjectTable::BasicGrid
        x._ensure_uniform_columns!
      end

      raise "Don't know how to append a #{x.class}" unless x.is_a?(ObjectTable::BasicGrid)
      next if x.empty?
      raise 'Mismatch in column names' unless (colnames | x.keys) == (colnames & x.keys)

      x.each do |k, v|
        new_values[k].push NArray.to_na(v)
      end
    end

    return self if new_values.empty?
    new_rows = new_values.values.first.map{|x| x.shape[-1]}.reduce(:+)
    return self unless (new_rows and new_rows != 0)
    new_rows += nrows

    new_values.each do |k, v|
      old_col = @columns[k]
      new_col = ObjectTable::Column.new(old_col.typecode, *old_col.shape[0...-1], new_rows)
      padding = [nil] * (old_col.rank - 1)
      new_col[*padding, 0 ... old_col.shape[-1]] = old_col

      row = old_col.shape[-1]
      v.each do |x|
        new_col[*padding, row ... (row + x.shape[-1])] = x
        row += x.shape[-1]
      end
      @columns[k] = new_col
    end
    self
  end

  def self.stack(*values)<commit_msg>use the new column stacking<commit_after>require_relative "object_table/version"
require_relative "object_table/basic_grid"
require_relative "object_table/table_methods"
require_relative "object_table/view"
require_relative "object_table/static_view"
require_relative "object_table/column"
require_relative "object_table/grouped"
require_relative "object_table/temp_grouped"

class ObjectTable
  include TableMethods

  attr_reader :columns

  def initialize(columns = {})
    super()

    unless columns.is_a? BasicGrid
      columns = BasicGrid[columns]
    end
    columns._ensure_uniform_columns!
    @columns = columns

    @columns.each do |k, v|
      @columns[k] = ObjectTable::Column.make(v)
    end
  end

  def add_column(name, typecode='object', *args)
    col = ObjectTable::Column.new(typecode, *args, nrows)
    columns[name] = col
  end

  def stack!(*others)
    new_values = Hash[colnames.zip(ncols.times.map{[]})]

    others.each do |x|
      case x
      when ObjectTable::TableMethods
        x = x.columns
      when ObjectTable::BasicGrid
        x._ensure_uniform_columns!
      end

      raise "Don't know how to append a #{x.class}" unless x.is_a?(ObjectTable::BasicGrid)
      next if x.empty?
      raise 'Mismatch in column names' unless (colnames | x.keys) == (colnames & x.keys)

      x.each do |k, v|
        new_values[k].push NArray.to_na(v)
      end
    end

    return self if new_values.empty?
    new_rows = new_values.values.first.map{|x| x.shape[-1]}.reduce(:+)
    return self unless (new_rows and new_rows != 0)
    new_rows += nrows

    new_values.each do |k, v|
      @columns[k] = @columns[k].stack(*v)
    end
    self
  end

  def self.stack(*values)
{'fraction_non_alphanumeric': 0.11615487316421896, 'fraction_numerical': 0.0032042723631508676, 'mean_word_length': 3.1995515695067263, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3905668', 'n_tokens_mistral': 1275, 'n_tokens_neox': 1239, 'n_words': 286}