<commit_before>require 'redshift_connector/exporter'
require 'redshift_connector/immediate_exporter'
require 'redshift_connector/importer'
require 'redshift_connector/data_file_bundle_params'
require 'redshift_connector/s3_data_file_bundle'
require 'redshift_connector/logger'

module RedshiftConnector
  class Connector
    def Connector.transport_delta_from_s3(
      bucket: nil,
      prefix:,
      format:,
      filter: nil,
      table:,
      columns:,
      delete_cond: nil,
      upsert_columns: nil,
      logger: RedshiftConnector.logger,
      quiet: false
    )
      logger = NullLogger.new if quiet
      bundle = S3DataFileBundle.for_prefix(
        bucket: (bucket ? S3Bucket.get(bucket) : S3Bucket.default),
        prefix: prefix,
        format: format,
        filter: filter,
        logger: logger
      )
      exporter = ImmediateExporter.new(bundle: bundle, logger: logger)
      importer = Importer.for_delta_upsert(
        table: table,
        columns: columns,
        delete_cond: delete_cond,
        upsert_columns: upsert_columns,
        logger: logger
      )
      new(exporter: exporter, importer: importer, logger: logger)
    end

    def Connector.transport_delta(
        schema:,
        table: nil,
        src_table: table,
        dest_table: table,
        condition:,
        columns:,
        delete_cond: nil,
        upsert_columns: nil,
        bucket: nil,
        txn_id: nil,
        filter:,
        logger: RedshiftConnector.logger,
        quiet: false
    )
      unless src_table and dest_table
        raise ArgumentError, "missing :table, :src_table or :dest_table"
      end
      logger = NullLogger.new if quiet
      bundle_params = DataFileBundleParams.new(
        bucket: bucket,
        schema: schema,
        table: src_table,
        txn_id: txn_id,
        filter: filter,
        logger: logger
      )
      exporter = Exporter.for_table_delta(
        bundle_params: bundle_params,
        schema: schema,
        table: src_table,
        columns: columns,
        condition: condition,
        logger: logger
      )
      importer = Importer.for_delta_upsert(
        table: dest_table,
        columns: columns,
        delete_cond: delete_cond,
        upsert_columns: upsert_columns,
        logger: logger
      )
      new(exporter: exporter, importer: importer, logger: logger)
    end

    def Connector.transport_all_from_s3(
        strategy: 'rename',
        table:,
        columns:,
        bucket: nil,
        prefix:,
        format:,
        filter: nil,
        logger: RedshiftConnector.logger,
        quiet: false
    )
      logger = NullLogger.new if quiet
      bundle = S3DataFileBundle.for_prefix(
        bucket: (bucket ? S3Bucket.get(bucket) : S3Bucket.default),
        prefix: prefix,
        format: format,
        filter: filter,
        logger: logger
      )
      exporter = ImmediateExporter.new(bundle: bundle, logger: logger)
      importer = Importer.for_rebuild(
        strategy: strategy,
        table: table,
        columns: columns,
        logger: logger
      )
      new(exporter: exporter, importer: importer, logger: logger)
    end

    def Connector.transport_all(
        strategy: 'rename',
        schema:,
        table:,
        src_table: table,
        dest_table: table,
        columns:,
        bucket: nil,
        txn_id: nil,
        filter:,
        logger: RedshiftConnector.logger,
        quiet: false
    )
      logger = NullLogger.new if quiet
      bundle_params = DataFileBundleParams.new(
        bucket: bucket,
        schema: schema,
        table: src_table,
        txn_id: txn_id,
        filter: filter,
        logger: logger
      )
      exporter = Exporter.for_table(
        bundle_params: bundle_params,<commit_msg>Set nil to table key<commit_after>require 'redshift_connector/exporter'
require 'redshift_connector/immediate_exporter'
require 'redshift_connector/importer'
require 'redshift_connector/data_file_bundle_params'
require 'redshift_connector/s3_data_file_bundle'
require 'redshift_connector/logger'

module RedshiftConnector
  class Connector
    def Connector.transport_delta_from_s3(
      bucket: nil,
      prefix:,
      format:,
      filter: nil,
      table:,
      columns:,
      delete_cond: nil,
      upsert_columns: nil,
      logger: RedshiftConnector.logger,
      quiet: false
    )
      logger = NullLogger.new if quiet
      bundle = S3DataFileBundle.for_prefix(
        bucket: (bucket ? S3Bucket.get(bucket) : S3Bucket.default),
        prefix: prefix,
        format: format,
        filter: filter,
        logger: logger
      )
      exporter = ImmediateExporter.new(bundle: bundle, logger: logger)
      importer = Importer.for_delta_upsert(
        table: table,
        columns: columns,
        delete_cond: delete_cond,
        upsert_columns: upsert_columns,
        logger: logger
      )
      new(exporter: exporter, importer: importer, logger: logger)
    end

    def Connector.transport_delta(
        schema:,
        table: nil,
        src_table: table,
        dest_table: table,
        condition:,
        columns:,
        delete_cond: nil,
        upsert_columns: nil,
        bucket: nil,
        txn_id: nil,
        filter:,
        logger: RedshiftConnector.logger,
        quiet: false
    )
      unless src_table and dest_table
        raise ArgumentError, "missing :table, :src_table or :dest_table"
      end
      logger = NullLogger.new if quiet
      bundle_params = DataFileBundleParams.new(
        bucket: bucket,
        schema: schema,
        table: src_table,
        txn_id: txn_id,
        filter: filter,
        logger: logger
      )
      exporter = Exporter.for_table_delta(
        bundle_params: bundle_params,
        schema: schema,
        table: src_table,
        columns: columns,
        condition: condition,
        logger: logger
      )
      importer = Importer.for_delta_upsert(
        table: dest_table,
        columns: columns,
        delete_cond: delete_cond,
        upsert_columns: upsert_columns,
        logger: logger
      )
      new(exporter: exporter, importer: importer, logger: logger)
    end

    def Connector.transport_all_from_s3(
        strategy: 'rename',
        table:,
        columns:,
        bucket: nil,
        prefix:,
        format:,
        filter: nil,
        logger: RedshiftConnector.logger,
        quiet: false
    )
      logger = NullLogger.new if quiet
      bundle = S3DataFileBundle.for_prefix(
        bucket: (bucket ? S3Bucket.get(bucket) : S3Bucket.default),
        prefix: prefix,
        format: format,
        filter: filter,
        logger: logger
      )
      exporter = ImmediateExporter.new(bundle: bundle, logger: logger)
      importer = Importer.for_rebuild(
        strategy: strategy,
        table: table,
        columns: columns,
        logger: logger
      )
      new(exporter: exporter, importer: importer, logger: logger)
    end

    def Connector.transport_all(
        strategy: 'rename',
        schema:,
        table: nil,
        src_table: table,
        dest_table: table,
        columns:,
        bucket: nil,
        txn_id: nil,
        filter:,
        logger: RedshiftConnector.logger,
        quiet: false
    )
      logger = NullLogger.new if quiet
      bundle_params = DataFileBundleParams.new(
        bucket: bucket,
        schema: schema,
        table: src_table,
        txn_id: txn_id,
        filter: filter,
        logger: logger
      )
      exporter = Exporter.for_table(
        bundle_params: bundle_params,
{'fraction_non_alphanumeric': 0.0795679093663549, 'fraction_numerical': 0.0023712290870768014, 'mean_word_length': 2.108927108927109, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4105411', 'n_tokens_mistral': 2210, 'n_tokens_neox': 2223, 'n_words': 526}