<commit_before>require 'spec_helper_min'
require 'factories/carto_visualizations'

describe Carto::OrganizationMetadataExportService do
  include NamedMapsHelper
  include Carto::Factories::Visualizations
  include TableSharing

  def create_organization_with_dependencies
    @sequel_organization = FactoryGirl.create(:organization_with_users)
    @organization = Carto::Organization.find(@sequel_organization.id)
    @owner = @organization.owner
    @non_owner = @organization.users.reject(&:organization_owner?).first

    @map, @table, @table_visualization, @visualization = create_full_visualization(@non_owner)
    share_visualization_with_user(@table_visualization, @owner)

    @asset = FactoryGirl.create(:carto_asset, organization: @organization)

    @organization.reload
  end

  def destroy_organization
    Organization[@organization.id].destroy_cascade
  end
    def import_organization_and_users_from_directory(path)
      # Import organization
      organization_file = Dir["#{path}/organization_*.json"].first
      organization = build_organization_from_json_export(File.read(organization_file))
      save_imported_organization(organization)

      user_list = Dir["#{path}/user_*"]

      # In order to get permissions right, we first import all users, then all datasets and finally, all maps
      organization.users = user_list.map do |user_path|
        Carto::UserMetadataExportService.new.import_user_from_directory(user_path, import_visualizations: false)
      end

      organization
    end

    def import_organization_visualizations_from_directory(organization, path)
      organization.users.each do |user|
        Carto::UserMetadataExportService.new.import_user_visualizations_from_directory(
          user, Carto::Visualization::TYPE_CANONICAL, "#{path}/user_#{user.id}"
        )
      end

      organization.users.each do |user|
        Carto::UserMetadataExportService.new.import_user_visualizations_from_directory(
          user, Carto::Visualization::TYPE_DERIVED, "#{path}/user_#{user.id}"
        )
      end

      organization
    end
  let(:service) { Carto::OrganizationMetadataExportService.new }

  describe '#organization export' do
    before(:all) do
      create_organization_with_dependencies
    end

    after(:all) do
      destroy_organization
    end

    it 'exports' do
      export = service.export_organization_json_hash(@organization.id)

      expect_export_matches_organization(export[:organization], @organization)
    end

    it 'includes all user model attributes' do
      export = service.export_organization_json_hash(@organization.id)

      expect(export[:organization].keys).to include(*@organization.attributes.symbolize_keys.keys)
    end
  end

  describe '#organization import' do
    it 'imports' do
      organization = service.build_organization_from_hash_export(full_export)

      expect_export_matches_organization(full_export[:organization], organization)
    end
  end

  describe '#organization export + import' do
    it 'export + import' do
      create_organization_with_dependencies
      export = service.export_organization_json_hash(@organization.id)
      expect_export_matches_organization(export[:organization], @organization)
      source_organization = @organization.attributes
      destroy_organization

      imported_organization = service.build_organization_from_hash_export(export)
      service.save_imported_organization(imported_organization)
      imported_organization.reload

      expect_export_matches_organization(export[:organization], imported_organization)
      compare_excluding_dates(imported_organization.attributes, source_organization)
    end
  end

  describe '#full export + import (organization, users and visualizations)' do
    it 'export + import organization, users and visualizations' do
      Dir.mktmpdir do |path|
        create_organization_with_dependencies
        service.export_organization_to_directory(@organization.id, path)
        source_organization = @organization.attributes
        source_users = @organization.users.map(&:attributes)

        # Destroy, keeping the database
        Table.any_instance.stubs(:remove_table_from_user_database)
        @organization.users.flat_map(&:visualizations).each(&:destroy)
        @organization.users.each(&:destroy)
        @organization.destroy

        imported_organization = service.import_organization_and_users_from_directory(path)
        import_organization_visualizations_from_directory(imported_organization, path)

        compare_excluding_dates(imported_organization.attributes, source_organization)
        expect(imported_organization.users.count).to eq source_users.count
        imported_organization.users.zip(source_users).each do |u1, u2|
          compare_excluding_dates(u1.attributes, u2)
        end
      end
    end
  end

  EXCLUDED_DATE_FIELDS = ['created_at', 'updated_at', 'period_end_date'].freeze

  def compare_excluding_dates(u1, u2)
    filtered1 = u1.reject { |k, _| EXCLUDED_DATE_FIELDS.include?(k) }
    filtered2 = u2.reject { |k, _| EXCLUDED_DATE_FIELDS.include?(k) }
    expect(filtered1).to eq filtered2
  end

  def expect_export_matches_organization(export, organization)
    Carto::OrganizationMetadataExportService::EXPORTED_ORGANIZATION_ATTRIBUTES.each do |att|
      expect(export[att]).to eq organization.attributes[att.to_s]
    end

    expect(export[:assets].count).to eq organization.assets.size
    export[:assets].zip(organization.assets).each { |exported_asset, asset| expect_export_matches_asset(exported_asset, asset) }
  end

  def expect_export_matches_asset(exported_asset, asset)
    expect(exported_asset[:public_url]).to eq asset.public_url
    expect(exported_asset[:kind]).to eq asset.kind<commit_msg>Remove extra code in spec :/<commit_after>require 'spec_helper_min'
require 'factories/carto_visualizations'

describe Carto::OrganizationMetadataExportService do
  include NamedMapsHelper
  include Carto::Factories::Visualizations
  include TableSharing

  def create_organization_with_dependencies
    @sequel_organization = FactoryGirl.create(:organization_with_users)
    @organization = Carto::Organization.find(@sequel_organization.id)
    @owner = @organization.owner
    @non_owner = @organization.users.reject(&:organization_owner?).first

    @map, @table, @table_visualization, @visualization = create_full_visualization(@non_owner)
    share_visualization_with_user(@table_visualization, @owner)

    @asset = FactoryGirl.create(:carto_asset, organization: @organization)

    @organization.reload
  end

  def destroy_organization
    Organization[@organization.id].destroy_cascade
  end

  let(:service) { Carto::OrganizationMetadataExportService.new }

  describe '#organization export' do
    before(:all) do
      create_organization_with_dependencies
    end

    after(:all) do
      destroy_organization
    end

    it 'exports' do
      export = service.export_organization_json_hash(@organization.id)

      expect_export_matches_organization(export[:organization], @organization)
    end

    it 'includes all user model attributes' do
      export = service.export_organization_json_hash(@organization.id)

      expect(export[:organization].keys).to include(*@organization.attributes.symbolize_keys.keys)
    end
  end

  describe '#organization import' do
    it 'imports' do
      organization = service.build_organization_from_hash_export(full_export)

      expect_export_matches_organization(full_export[:organization], organization)
    end
  end

  describe '#organization export + import' do
    it 'export + import' do
      create_organization_with_dependencies
      export = service.export_organization_json_hash(@organization.id)
      expect_export_matches_organization(export[:organization], @organization)
      source_organization = @organization.attributes
      destroy_organization

      imported_organization = service.build_organization_from_hash_export(export)
      service.save_imported_organization(imported_organization)
      imported_organization.reload

      expect_export_matches_organization(export[:organization], imported_organization)
      compare_excluding_dates(imported_organization.attributes, source_organization)
    end
  end

  describe '#full export + import (organization, users and visualizations)' do
    it 'export + import organization, users and visualizations' do
      Dir.mktmpdir do |path|
        create_organization_with_dependencies
        service.export_organization_to_directory(@organization.id, path)
        source_organization = @organization.attributes
        source_users = @organization.users.map(&:attributes)

        # Destroy, keeping the database
        Table.any_instance.stubs(:remove_table_from_user_database)
        @organization.users.flat_map(&:visualizations).each(&:destroy)
        @organization.users.each(&:destroy)
        @organization.destroy

        imported_organization = service.import_organization_and_users_from_directory(path)
        service.import_organization_visualizations_from_directory(imported_organization, path)

        compare_excluding_dates(imported_organization.attributes, source_organization)
        expect(imported_organization.users.count).to eq source_users.count
        imported_organization.users.zip(source_users).each do |u1, u2|
          compare_excluding_dates(u1.attributes, u2)
        end
      end
    end
  end

  EXCLUDED_DATE_FIELDS = ['created_at', 'updated_at', 'period_end_date'].freeze

  def compare_excluding_dates(u1, u2)
    filtered1 = u1.reject { |k, _| EXCLUDED_DATE_FIELDS.include?(k) }
    filtered2 = u2.reject { |k, _| EXCLUDED_DATE_FIELDS.include?(k) }
    expect(filtered1).to eq filtered2
  end

  def expect_export_matches_organization(export, organization)
    Carto::OrganizationMetadataExportService::EXPORTED_ORGANIZATION_ATTRIBUTES.each do |att|
      expect(export[att]).to eq organization.attributes[att.to_s]
    end

    expect(export[:assets].count).to eq organization.assets.size
    export[:assets].zip(organization.assets).each { |exported_asset, asset| expect_export_matches_asset(exported_asset, asset) }
  end

  def expect_export_matches_asset(exported_asset, asset)
    expect(exported_asset[:public_url]).to eq asset.public_url
    expect(exported_asset[:kind]).to eq asset.kind
{'fraction_non_alphanumeric': 0.08297069915868871, 'fraction_numerical': 0.0023208587177255585, 'mean_word_length': 4.6886688668866885, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2560549', 'n_tokens_mistral': 3043, 'n_tokens_neox': 2809, 'n_words': 496}