<commit_before>require 'dump/snapshot'
require 'dump/archive_tar_minitar'
require 'dump/assets'
require 'progress'
require 'rake'
require 'zlib'
require 'tempfile'

module Dump
  # Creating dump
  class Writer < Snapshot
    attr_reader :stream, :config

    def self.create(path)
      new(path).open do |dump|
        dump.silence do
          dump.write_schema

          dump.write_tables
          dump.write_assets

          dump.write_config
        end
      end
    end

    def open
      Pathname.new(path).dirname.mkpath
      Zlib::GzipWriter.open(path) do |gzip|
        gzip.mtime = Time.utc(2000)
        lock do
          Archive::Tar::Minitar.open(gzip, 'w') do |stream|
            @stream = stream
            @config = {:tables => {}}
            yield(self)
          end
        end
      end
    end

    def create_file(name)
      Tempfile.open('dump') do |temp|
        yield(temp)
        temp.open
        stream.tar.add_file_simple(name, :mode => 0o100444, :size => temp.length) do |f|
          f.write(temp.read(4096)) until temp.eof?
        end
      end
    end

    def write_schema
      create_file('schema.rb') do |f|
        Dump::Env.with_env('SCHEMA' => f.path) do
          Rake::Task['db:schema:dump'].invoke
        end
      end
    end

    def write_tables
      verify_connection
      tables_to_dump.with_progress('Tables') do |table|
        write_table(table)
      end
    end

    def write_table(table)
      row_count = table_row_count(table)
      config[:tables][table] = row_count
      Progress.start(table, 1 + row_count) do
        create_file("#{table}.dump") do |f|
          columns = table_columns(table)
          column_names = columns.map(&:name).sort
          columns_by_name = columns.index_by(&:name)

          Marshal.dump(column_names, f)
          Progress.step

          type_cast = if columns.first.respond_to?(:type_cast_from_database)
            proc do |column_name, value|
              columns_by_name[column_name].type_cast_from_database(value)
            end
          else
            proc do |column_name, value|
              columns_by_name[column_name].type_cast(value)
            end
          end

          each_table_row(table, row_count) do |row|
            values = column_names.map do |column_name|
              type_cast.call(column_name, row[column_name])
            end
            Marshal.dump(values, f)
            Progress.step
          end
        end
      end
    end

    def write_assets
      assets = assets_to_dump
      return if assets.blank?

      config[:assets] = {}
      Dir.chdir(Dump.rails_root) do<commit_msg>add rails 5 way of casting values from database to ruby types<commit_after>require 'dump/snapshot'
require 'dump/archive_tar_minitar'
require 'dump/assets'
require 'progress'
require 'rake'
require 'zlib'
require 'tempfile'

module Dump
  # Creating dump
  class Writer < Snapshot
    attr_reader :stream, :config

    def self.create(path)
      new(path).open do |dump|
        dump.silence do
          dump.write_schema

          dump.write_tables
          dump.write_assets

          dump.write_config
        end
      end
    end

    def open
      Pathname.new(path).dirname.mkpath
      Zlib::GzipWriter.open(path) do |gzip|
        gzip.mtime = Time.utc(2000)
        lock do
          Archive::Tar::Minitar.open(gzip, 'w') do |stream|
            @stream = stream
            @config = {:tables => {}}
            yield(self)
          end
        end
      end
    end

    def create_file(name)
      Tempfile.open('dump') do |temp|
        yield(temp)
        temp.open
        stream.tar.add_file_simple(name, :mode => 0o100444, :size => temp.length) do |f|
          f.write(temp.read(4096)) until temp.eof?
        end
      end
    end

    def write_schema
      create_file('schema.rb') do |f|
        Dump::Env.with_env('SCHEMA' => f.path) do
          Rake::Task['db:schema:dump'].invoke
        end
      end
    end

    def write_tables
      verify_connection
      tables_to_dump.with_progress('Tables') do |table|
        write_table(table)
      end
    end

    def write_table(table)
      row_count = table_row_count(table)
      config[:tables][table] = row_count
      Progress.start(table, 1 + row_count) do
        create_file("#{table}.dump") do |f|
          columns = table_columns(table)
          column_names = columns.map(&:name).sort
          columns_by_name = columns.index_by(&:name)

          Marshal.dump(column_names, f)
          Progress.step

          type_cast = case
          when columns.first.respond_to?(:type_cast_from_database)
            proc do |column_name, value|
              columns_by_name[column_name].type_cast_from_database(value)
            end
          when columns.first.respond_to?(:type_cast)
            proc do |column_name, value|
              columns_by_name[column_name].type_cast(value)
            end
          when connection.respond_to?(:lookup_cast_type_from_column)
            proc do |column_name, value|
              connection.lookup_cast_type_from_column(columns_by_name[column_name]).deserialize(value)
            end
          else
            fail 'Failed to determine the way to convert values from database input to the appropriate ruby type'
          end

          each_table_row(table, row_count) do |row|
            values = column_names.map do |column_name|
              type_cast.call(column_name, row[column_name])
            end
            Marshal.dump(values, f)
            Progress.step
          end
        end
      end
    end

    def write_assets
      assets = assets_to_dump
      return if assets.blank?

      config[:assets] = {}
      Dir.chdir(Dump.rails_root) do
{'fraction_non_alphanumeric': 0.09225413402959094, 'fraction_numerical': 0.005744125326370757, 'mean_word_length': 1.9942678478374154, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5959629', 'n_tokens_mistral': 1786, 'n_tokens_neox': 1728, 'n_words': 376}