<commit_before># frozen_string_literal: true

require_dependency "gobierto_data"

module GobiertoData
  class Dataset < ApplicationRecord
    include GobiertoCommon::Sluggable
    include GobiertoData::Favoriteable

    belongs_to :site
    has_many :queries, dependent: :destroy, class_name: "GobiertoData::Query"
    has_many :visualizations, through: :queries, class_name: "GobiertoData::Visualization"

    translates :name

    enum visibility_level: { draft: 0, active: 1 }

    validates :site, :name, :slug, :table_name, presence: true
    validates(
      :table_name,
      format: {
        with: /\A[a-zA-Z_][a-zA-Z0-9_]+\z/,
        message: I18n.t("errors.messages.invalid_table_name")
      },
      length: { maximum: 50 }
    )
    validates :slug, uniqueness: { scope: :site_id }

    def attributes_for_slug
      [name]
    end

    def available_formats
      [:json, :csv, :xlsx]
    end

    def rails_model
      @rails_model ||= begin
                         return Connection.const_get(internal_rails_class_name) if Connection.const_defined?(internal_rails_class_name)

                         db_config = Connection.db_config(site)
                         return if db_config.blank?

                         db_config = db_config.fetch(:read_db_config, db_config)

                         Class.new(Connection).tap do |connection_model|
                           Connection.const_set(internal_rails_class_name, connection_model)
                           connection_model.establish_connection(db_config)
                           connection_model.table_name = table_name
                         end
                       end
    end

    def load_data_from_file(file_path, schema_file: nil, csv_separator: ",", append: false)
      schema = if schema_file.blank?
                 {}
               elsif schema_file.is_a? Hash
                 schema_file.deep_symbolize_keys
               else
                 JSON.parse(File.read(schema_file)).deep_symbolize_keys
               end
      statements = GobiertoData::Datasets::CreationStatements.new(
        self,
        file_path,
        schema,
        csv_separator: csv_separator,
        append: append,
        use_stdin: true
      )

      query_result = Connection.execute_write_query_from_file_using_stdin(site, statements.sql_code, file_path: file_path)
      touch(:data_updated_at) unless query_result.has_key?(:errors)
      {
        db_result: query_result,
        schema: statements.schema,
        script: statements.transaction_sql_code
      }
    end

    private

    def internal_rails_class_name
      @internal_rails_class_name ||= "site_id_#{site.id}_table_#{table_name}".classify
    end
  end
end
<commit_msg>Add callback to set schema of datasets<commit_after># frozen_string_literal: true

require_dependency "gobierto_data"

module GobiertoData
  class Dataset < ApplicationRecord
    include GobiertoCommon::Sluggable
    include GobiertoData::Favoriteable

    belongs_to :site
    has_many :queries, dependent: :destroy, class_name: "GobiertoData::Query"
    has_many :visualizations, through: :queries, class_name: "GobiertoData::Visualization"

    translates :name

    enum visibility_level: { draft: 0, active: 1 }

    validates :site, :name, :slug, :table_name, presence: true
    validates(
      :table_name,
      format: {
        with: /\A[a-zA-Z_][a-zA-Z0-9_]+\z/,
        message: I18n.t("errors.messages.invalid_table_name")
      },
      length: { maximum: 50 }
    )
    validates :slug, uniqueness: { scope: :site_id }

    before_save :set_schema, if: :will_save_change_to_visibility_level?

    def attributes_for_slug
      [name]
    end

    def available_formats
      [:json, :csv, :xlsx]
    end

    def rails_model
      @rails_model ||= begin
                         return Connection.const_get(internal_rails_class_name) if Connection.const_defined?(internal_rails_class_name)

                         db_config = Connection.db_config(site)
                         return if db_config.blank?

                         db_config = db_config.fetch(:read_db_config, db_config)

                         Class.new(Connection).tap do |connection_model|
                           Connection.const_set(internal_rails_class_name, connection_model)
                           connection_model.establish_connection(db_config)
                           connection_model.table_name = table_name
                         end
                       end
    end

    def load_data_from_file(file_path, schema_file: nil, csv_separator: ",", append: false)
      schema = if schema_file.blank?
                 {}
               elsif schema_file.is_a? Hash
                 schema_file.deep_symbolize_keys
               else
                 JSON.parse(File.read(schema_file)).deep_symbolize_keys
               end
      statements = GobiertoData::Datasets::CreationStatements.new(
        self,
        file_path,
        schema,
        csv_separator: csv_separator,
        append: append,
        use_stdin: true
      )

      query_result = Connection.execute_write_query_from_file_using_stdin(site, statements.sql_code, file_path: file_path)
      set_schema
      touch(:data_updated_at) unless query_result.has_key?(:errors)
      {
        db_result: query_result,
        schema: statements.schema,
        script: statements.transaction_sql_code
      }
    end

    private

    def set_schema
      if draft?
        Connection.execute_query(site, "ALTER TABLE IF EXISTS #{table_name} SET SCHEMA draft", write: true)
      else
        Connection.execute_query(site, "ALTER TABLE IF EXISTS #{table_name} SET SCHEMA public", write: true)
      end
    end

    def internal_rails_class_name
      @internal_rails_class_name ||= "site_id_#{site.id}_table_#{table_name}".classify
    end
  end
end

{'fraction_non_alphanumeric': 0.07987711213517665, 'fraction_numerical': 0.00273084144051886, 'mean_word_length': 2.2939853850477796, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4919975', 'n_tokens_mistral': 1780, 'n_tokens_neox': 1694, 'n_words': 332}