<commit_before># encoding: utf-8

require_relative 'connector/fdw_support'
require_relative 'connector/errors'
require_relative 'connector/providers'
require_relative 'connector/parameters'

module Carto
  # This class provides remote database connection services based on FDW
  class Connector

    attr_reader :provider_name

    def initialize(parameters, options = {})
      @logger     = options[:logger]
      @user       = options[:user]

      @unique_suffix = UUIDTools::UUID.timestamp_create.to_s.delete('-') # .to_i.to_s(16) # or hash from user, etc.
      @params = Parameters.new(parameters)

      @provider_name = @params[:provider]
      @provider_name ||= DEFAULT_PROVIDER

      raise InvalidParametersError.new(message: "Provider not defined") if @provider_name.blank?
      @provider = Connector.provider_class(@provider_name).try :new, @params
      raise InvalidParametersError.new(message: "Invalid provider", provider: @provider_name) if @provider.blank?
    end

    def copy_table(schema_name:, table_name:)
      log "Connector Copy table  #{schema_name}.#{table_name}"
      validate!
      # TODO: logging with CartoDB::Logger
      with_server do
        begin
          qualified_table_name = %{"#{schema_name}"."#{table_name}"}
          foreign_table_name = @provider.foreign_table_name(foreign_prefix)
          log "Creating Foreign Table"
          execute_as_superuser create_foreign_table_command
          log "Copying Foreign Table"
          max_rows = limits[:max_rows]
          execute copy_foreign_table_command(
            qualified_table_name, qualified_foreign_table_name(foreign_table_name), max_rows
          )
          check_copied_table_size(qualified_table_name, max_rows)
        ensure
          execute_as_superuser drop_foreign_table_command(foreign_table_name) if foreign_table_name
        end
      end
    end

    def list_tables
      validate! only: [:connection]
      with_server do
        execute %{
          SELECT * FROM ODBCTablesList('#{server_name}');
        }
      end
    end

    def remote_data_updated?
      # TODO: can we detect if query results have changed?
      true
    end

    def remote_table_name
      @provider.table_name
    end

    # General availabillity check
    def self.check_availability!(user)
      unless user.has_feature_flag?('carto-connectors')
        raise ConnectorsDisabledError.new(user: user)
      end
    end

    # Check availability for a user and provider
    def check_availability!
      Connector.check_availability!(@user)
      if !enabled?
        raise ConnectorsDisabledError.new(user: @user, provider: @provider_name)
      end
    end

    # Limits for the user/provider
    def limits
      Connector.limits provider_name: @provider_name, user: @user
    end

    # Availability for the user/provider
    def enabled?
      limits[:enabled]
    end

    def self.limits(provider_name:, user:)
      if configuration = user.connector_configuration(provider_name)
        { enabled: configuration.enabled?, max_rows: configuration.max_rows }
      else
        {}
      end
    end

    # Information about a connector's features and parameters.
    #
    # Example:
    # {
    #   'mysql' => {
    #     features: {
    #                 "sql_queries":    true,
    #                 "list_databases": false,
    #                 "list_tables":    true,
    #                 "preview_table":  false
    #     },
    #     parameters: {
    #       connection: {
    #         server: { required: true, description: "..." },
    #         ...
    #       },
    #       table: { required: true, description: "..." },
    #       ...
    #     }
    #   }, ...
    # }
    def self.information(provider_name, user = nil)
      provider = provider_class(provider_name)
      raise InvalidParametersError.new(message: "Invalid provider", provider: provider_name) if provider.blank?
      provider.information
    end

    # Available providers information.
    #
    # Example:
    # {
    #   'mysql' => { name: 'MySQL', description: '...', enabled: true },
    #   ...
    # }
    def self.providers(user = nil)
      providers_info = {}
      provider_ids.each do |id|
        next unless provider_public?(id)
        # TODO: load description template for provider id
        description = nil
        if user
          enabled = Connector.limits(user: user, provider_name: id)[:enabled]
        end
        providers_info[id] = {
          name:        provider_name(id),
          description: description,
          enabled:     enabled
        }
      end
      providers_info
    end

    private

    # Execute code that requires a FDW server/user mapping
    # The server name is given by the method `#server_name`
    def with_server
      # Currently we create temporary server and user mapings when we need them,
      # and drop them after use.
      log "Creating Server"
      execute_as_superuser create_server_command
      log "Creating Usermap"
      execute_as_superuser create_usermap_command
      yield
    rescue => error
      log "Connector Error #{error}"
      raise error
    ensure
      log "Connector cleanup"
      execute_as_superuser drop_usermap_command
      execute_as_superuser drop_server_command
      log "Connector cleaned-up"
    end

    # Validate parameters.
    # An array of parameter names to validate can be passed via :only.
    # By default all parameters are validated
    def validate!(only: nil)
      @provider.validate!(only: only)
    end

    def log(message, truncate = true)
      @logger.append message, truncate if @logger
    end

    MAX_PG_IDENTIFIER_LEN = 60
    MIN_TAB_ID_LEN        = 10

    def server_name
      max_len = MAX_PG_IDENTIFIER_LEN - @unique_suffix.size - MIN_TAB_ID_LEN - 1
      connector_name = Carto::DB::Sanitize.sanitize_identifier @provider_name
      connector_name[0...max_len]
      "#{connector_name.downcase}_#{@unique_suffix}"
    end

    def foreign_prefix
      "#{server_name}_"
    end

    def foreign_table_schema
      # since connectors' foreign table names are unique (because
      # server names are unique and not reused)
      # we could in principle use any schema (@schema, 'public', 'cdb_importer')
      CartoDB::Connector::Importer::ORIGIN_SCHEMA
    end<commit_msg>Fix Connector server name and foreign table prefix<commit_after># encoding: utf-8

require_relative 'connector/fdw_support'
require_relative 'connector/errors'
require_relative 'connector/providers'
require_relative 'connector/parameters'

module Carto
  # This class provides remote database connection services based on FDW
  class Connector

    attr_reader :provider_name

    def initialize(parameters, options = {})
      @logger     = options[:logger]
      @user       = options[:user]

      @unique_suffix = UUIDTools::UUID.timestamp_create.to_s.delete('-') # .to_i.to_s(16) # or hash from user, etc.
      @params = Parameters.new(parameters)

      @provider_name = @params[:provider]
      @provider_name ||= DEFAULT_PROVIDER

      raise InvalidParametersError.new(message: "Provider not defined") if @provider_name.blank?
      @provider = Connector.provider_class(@provider_name).try :new, @params
      raise InvalidParametersError.new(message: "Invalid provider", provider: @provider_name) if @provider.blank?
    end

    def copy_table(schema_name:, table_name:)
      log "Connector Copy table  #{schema_name}.#{table_name}"
      validate!
      # TODO: logging with CartoDB::Logger
      with_server do
        begin
          qualified_table_name = %{"#{schema_name}"."#{table_name}"}
          foreign_table_name = @provider.foreign_table_name(foreign_prefix)
          log "Creating Foreign Table"
          execute_as_superuser create_foreign_table_command
          log "Copying Foreign Table"
          max_rows = limits[:max_rows]
          execute copy_foreign_table_command(
            qualified_table_name, qualified_foreign_table_name(foreign_table_name), max_rows
          )
          check_copied_table_size(qualified_table_name, max_rows)
        ensure
          execute_as_superuser drop_foreign_table_command(foreign_table_name) if foreign_table_name
        end
      end
    end

    def list_tables
      validate! only: [:connection]
      with_server do
        execute %{
          SELECT * FROM ODBCTablesList('#{server_name}');
        }
      end
    end

    def remote_data_updated?
      # TODO: can we detect if query results have changed?
      true
    end

    def remote_table_name
      @provider.table_name
    end

    # General availabillity check
    def self.check_availability!(user)
      unless user.has_feature_flag?('carto-connectors')
        raise ConnectorsDisabledError.new(user: user)
      end
    end

    # Check availability for a user and provider
    def check_availability!
      Connector.check_availability!(@user)
      if !enabled?
        raise ConnectorsDisabledError.new(user: @user, provider: @provider_name)
      end
    end

    # Limits for the user/provider
    def limits
      Connector.limits provider_name: @provider_name, user: @user
    end

    # Availability for the user/provider
    def enabled?
      limits[:enabled]
    end

    def self.limits(provider_name:, user:)
      if configuration = user.connector_configuration(provider_name)
        { enabled: configuration.enabled?, max_rows: configuration.max_rows }
      else
        {}
      end
    end

    # Information about a connector's features and parameters.
    #
    # Example:
    # {
    #   'mysql' => {
    #     features: {
    #                 "sql_queries":    true,
    #                 "list_databases": false,
    #                 "list_tables":    true,
    #                 "preview_table":  false
    #     },
    #     parameters: {
    #       connection: {
    #         server: { required: true, description: "..." },
    #         ...
    #       },
    #       table: { required: true, description: "..." },
    #       ...
    #     }
    #   }, ...
    # }
    def self.information(provider_name, user = nil)
      provider = provider_class(provider_name)
      raise InvalidParametersError.new(message: "Invalid provider", provider: provider_name) if provider.blank?
      provider.information
    end

    # Available providers information.
    #
    # Example:
    # {
    #   'mysql' => { name: 'MySQL', description: '...', enabled: true },
    #   ...
    # }
    def self.providers(user = nil)
      providers_info = {}
      provider_ids.each do |id|
        next unless provider_public?(id)
        # TODO: load description template for provider id
        description = nil
        if user
          enabled = Connector.limits(user: user, provider_name: id)[:enabled]
        end
        providers_info[id] = {
          name:        provider_name(id),
          description: description,
          enabled:     enabled
        }
      end
      providers_info
    end

    private

    # Execute code that requires a FDW server/user mapping
    # The server name is given by the method `#server_name`
    def with_server
      # Currently we create temporary server and user mapings when we need them,
      # and drop them after use.
      log "Creating Server"
      execute_as_superuser create_server_command
      log "Creating Usermap"
      execute_as_superuser create_usermap_command
      yield
    rescue => error
      log "Connector Error #{error}"
      raise error
    ensure
      log "Connector cleanup"
      execute_as_superuser drop_usermap_command
      execute_as_superuser drop_server_command
      log "Connector cleaned-up"
    end

    # Validate parameters.
    # An array of parameter names to validate can be passed via :only.
    # By default all parameters are validated
    def validate!(only: nil)
      @provider.validate!(only: only)
    end

    def log(message, truncate = true)
      @logger.append message, truncate if @logger
    end

    # maximum unique identifier length in PostgreSQL
    MAX_PG_IDENTIFIER_LEN = 63
    # minimum length left available for the table part in foreign table names
    MIN_TAB_ID_LEN        = 10

    # Named used for the foreign server (unique poer Connector instance)
    def server_name
      max_len = MAX_PG_IDENTIFIER_LEN - @unique_suffix.size - MIN_TAB_ID_LEN - 1
      connector_name = Carto::DB::Sanitize.sanitize_identifier @provider_name
      "#{connector_name[0...max_len].downcase}_#{@unique_suffix}"
    end

    # Prefix to be used by foreign table names (so they're unique per Connector instance)
    # This leaves at least MIN_TAB_ID_LEN available identifier characters given PostgreSQL's
    # limit of MAX_PG_IDENTIFIER_LEN
    def foreign_prefix
      "#{server_name}_"
    end

    def foreign_table_schema
      # since connectors' foreign table names are unique (because
      # server names are unique and not reused)
      # we could in principle use any schema (@schema, 'public', 'cdb_importer')
      CartoDB::Connector::Importer::ORIGIN_SCHEMA
    end
{'fraction_non_alphanumeric': 0.0832313341493268, 'fraction_numerical': 0.0013769889840881274, 'mean_word_length': 2.5180301399354144, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5287577', 'n_tokens_mistral': 3739, 'n_tokens_neox': 3660, 'n_words': 968}