<commit_before>require 'json'
require 'carto/export/layer_exporter'
require 'carto/export/data_import_exporter'
require_dependency 'carto/export/connector_configuration_exporter'

# Not migrated
# likes -> difficult to do between clouds
# snapshots -> difficult to do between clouds, not in use yet
# tags -> regenerated from tables
# visualization_export -> only purpose would be logging

# Version History
# 1.0.0: export user metadata
# 1.0.1: export search tweets
# 1.0.2: export user notifications
# 1.0.3: export rate limits
# 1.0.4: company and phone in users table
# 1.0.5: synchronization_oauths and connector configurations
# 1.0.6: client_applications & friends and sql_copy rate_limits

module Carto
  module UserMetadataExportServiceConfiguration
    CURRENT_VERSION = '1.0.6'.freeze
    EXPORTED_USER_ATTRIBUTES = [
      :email, :crypted_password, :salt, :database_name, :username, :admin, :enabled, :invite_token, :invite_token_date,
      :map_enabled, :quota_in_bytes, :table_quota, :account_type, :private_tables_enabled, :period_end_date,
      :map_view_quota, :max_layers, :database_timeout, :user_timeout, :upgraded_at, :map_view_block_price,
      :geocoding_quota, :dashboard_viewed_at, :sync_tables_enabled, :database_host, :geocoding_block_price, :api_key,
      :notification, :organization_id, :created_at, :updated_at, :disqus_shortname, :id, :twitter_username, :website,
      :description, :name, :avatar_url, :database_schema, :soft_geocoding_limit, :auth_token,
      :twitter_datasource_enabled, :twitter_datasource_block_price, :twitter_datasource_block_size,
      :twitter_datasource_quota, :soft_twitter_datasource_limit, :available_for_hire, :private_maps_enabled,
      :google_sign_in, :last_password_change_date, :max_import_file_size, :max_import_table_row_count,
      :max_concurrent_import_count, :last_common_data_update_date, :google_maps_key, :google_maps_private_key,
      :enable_account_token, :location, :here_isolines_quota, :here_isolines_block_price, :soft_here_isolines_limit,
      :obs_snapshot_quota, :obs_snapshot_block_price, :soft_obs_snapshot_limit, :mobile_xamarin,
      :mobile_custom_watermark, :mobile_offline_maps, :mobile_gis_extension, :mobile_max_open_users,
      :mobile_max_private_users, :obs_general_quota, :obs_general_block_price, :soft_obs_general_limit, :viewer,
      :salesforce_datasource_enabled, :builder_enabled, :geocoder_provider, :isolines_provider, :routing_provider,
      :github_user_id, :engine_enabled, :mapzen_routing_quota, :mapzen_routing_block_price, :soft_mapzen_routing_limit,
      :no_map_logo, :org_admin, :last_name, :user_render_timeout, :database_render_timeout, :frontend_version,
      :asset_host, :state, :company, :phone, :industry, :job_role
    ].freeze

    BLANK_UUID = '00000000-0000-0000-0000-000000000000'.freeze

    def compatible_version?(version)
      version.to_i == CURRENT_VERSION.split('.')[0].to_i
    end
  end

  module UserMetadataExportServiceImporter
    include UserMetadataExportServiceConfiguration
    include LayerImporter
    include DataImportImporter
    include ConnectorConfigurationImporter

    def build_user_from_json_export(exported_json_string)
      build_user_from_hash_export(parse_json(exported_json_string))
    end

    def build_user_from_hash_export(exported_hash)
      raise 'Wrong export version' unless compatible_version?(exported_hash[:version])

      build_user_from_hash(exported_hash[:user])
    end

    def build_search_tweets_from_json_export(exported_json_string)
      build_search_tweets_from_hash_export(parse_json(exported_json_string))
    end

    def build_search_tweets_from_hash_export(exported_hash)
      exported_hash[:user].fetch(:search_tweets, []).map { |st| build_search_tweet_from_hash(st) }
    end

    def save_imported_user(user)
      user.save!
      ::User[user.id].after_save
      user.client_applications.first.access_tokens.each do |t|
        # AR does not know about this, so it needs to be fixed
        t.update_column(:type, 'AccessToken')
        AccessToken[t.id].after_save
      end
    end

    def save_imported_search_tweet(search_tweet, user)
      if search_tweet.data_import
        persisted_import = Carto::DataImport.where(id: search_tweet.data_import.id).first
        search_tweet.data_import = persisted_import if persisted_import
        search_tweet.table_id = search_tweet.data_import.table_id
      else
        # Some search tweets can be exported without data import if the FK point to a non-existent data import.
        # However, this field is NOT NULL, so we cannot leave it empty.
        # We could skip the import of the tweet, but instead, we keep it with an invalid ID (like in the source), so
        # we can still correctly compute quota usage.
        search_tweet.data_import_id = BLANK_UUID
      end
      search_tweet.user = user
      search_tweet.save!
    end

    private

    def parse_json(exported_json_string)
      JSON.parse(exported_json_string, symbolize_names: true)
    end<commit_msg>Fix save imported user if client application not passed in<commit_after>require 'json'
require 'carto/export/layer_exporter'
require 'carto/export/data_import_exporter'
require_dependency 'carto/export/connector_configuration_exporter'

# Not migrated
# likes -> difficult to do between clouds
# snapshots -> difficult to do between clouds, not in use yet
# tags -> regenerated from tables
# visualization_export -> only purpose would be logging

# Version History
# 1.0.0: export user metadata
# 1.0.1: export search tweets
# 1.0.2: export user notifications
# 1.0.3: export rate limits
# 1.0.4: company and phone in users table
# 1.0.5: synchronization_oauths and connector configurations
# 1.0.6: client_applications & friends and sql_copy rate_limits

module Carto
  module UserMetadataExportServiceConfiguration
    CURRENT_VERSION = '1.0.6'.freeze
    EXPORTED_USER_ATTRIBUTES = [
      :email, :crypted_password, :salt, :database_name, :username, :admin, :enabled, :invite_token, :invite_token_date,
      :map_enabled, :quota_in_bytes, :table_quota, :account_type, :private_tables_enabled, :period_end_date,
      :map_view_quota, :max_layers, :database_timeout, :user_timeout, :upgraded_at, :map_view_block_price,
      :geocoding_quota, :dashboard_viewed_at, :sync_tables_enabled, :database_host, :geocoding_block_price, :api_key,
      :notification, :organization_id, :created_at, :updated_at, :disqus_shortname, :id, :twitter_username, :website,
      :description, :name, :avatar_url, :database_schema, :soft_geocoding_limit, :auth_token,
      :twitter_datasource_enabled, :twitter_datasource_block_price, :twitter_datasource_block_size,
      :twitter_datasource_quota, :soft_twitter_datasource_limit, :available_for_hire, :private_maps_enabled,
      :google_sign_in, :last_password_change_date, :max_import_file_size, :max_import_table_row_count,
      :max_concurrent_import_count, :last_common_data_update_date, :google_maps_key, :google_maps_private_key,
      :enable_account_token, :location, :here_isolines_quota, :here_isolines_block_price, :soft_here_isolines_limit,
      :obs_snapshot_quota, :obs_snapshot_block_price, :soft_obs_snapshot_limit, :mobile_xamarin,
      :mobile_custom_watermark, :mobile_offline_maps, :mobile_gis_extension, :mobile_max_open_users,
      :mobile_max_private_users, :obs_general_quota, :obs_general_block_price, :soft_obs_general_limit, :viewer,
      :salesforce_datasource_enabled, :builder_enabled, :geocoder_provider, :isolines_provider, :routing_provider,
      :github_user_id, :engine_enabled, :mapzen_routing_quota, :mapzen_routing_block_price, :soft_mapzen_routing_limit,
      :no_map_logo, :org_admin, :last_name, :user_render_timeout, :database_render_timeout, :frontend_version,
      :asset_host, :state, :company, :phone, :industry, :job_role
    ].freeze

    BLANK_UUID = '00000000-0000-0000-0000-000000000000'.freeze

    def compatible_version?(version)
      version.to_i == CURRENT_VERSION.split('.')[0].to_i
    end
  end

  module UserMetadataExportServiceImporter
    include UserMetadataExportServiceConfiguration
    include LayerImporter
    include DataImportImporter
    include ConnectorConfigurationImporter

    def build_user_from_json_export(exported_json_string)
      build_user_from_hash_export(parse_json(exported_json_string))
    end

    def build_user_from_hash_export(exported_hash)
      raise 'Wrong export version' unless compatible_version?(exported_hash[:version])

      build_user_from_hash(exported_hash[:user])
    end

    def build_search_tweets_from_json_export(exported_json_string)
      build_search_tweets_from_hash_export(parse_json(exported_json_string))
    end

    def build_search_tweets_from_hash_export(exported_hash)
      exported_hash[:user].fetch(:search_tweets, []).map { |st| build_search_tweet_from_hash(st) }
    end

    def save_imported_user(user)
      user.save!
      ::User[user.id].after_save

      client_application = user.client_applications.first
      if client_application
        client_application.access_tokens.each do |t|
          # AR does not know about this, so it needs to be fixed
          t.update_column(:type, 'AccessToken')
          AccessToken[t.id].after_save
        end
      end
    end

    def save_imported_search_tweet(search_tweet, user)
      if search_tweet.data_import
        persisted_import = Carto::DataImport.where(id: search_tweet.data_import.id).first
        search_tweet.data_import = persisted_import if persisted_import
        search_tweet.table_id = search_tweet.data_import.table_id
      else
        # Some search tweets can be exported without data import if the FK point to a non-existent data import.
        # However, this field is NOT NULL, so we cannot leave it empty.
        # We could skip the import of the tweet, but instead, we keep it with an invalid ID (like in the source), so
        # we can still correctly compute quota usage.
        search_tweet.data_import_id = BLANK_UUID
      end
      search_tweet.user = user
      search_tweet.save!
    end

    private

    def parse_json(exported_json_string)
      JSON.parse(exported_json_string, symbolize_names: true)
    end
{'fraction_non_alphanumeric': 0.07983193277310924, 'fraction_numerical': 0.011139339456712917, 'mean_word_length': 4.905943450663589, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '669881', 'n_tokens_mistral': 3397, 'n_tokens_neox': 3263, 'n_words': 677}