<commit_before># encoding: utf-8
require_relative '../spec_helper'
require_relative 'data_import_shared_examples'

describe DataImport do
  let(:data_import_class) { ::DataImport }
  it_behaves_like 'DataImport model'

  before(:each) do
    @user = create_user
    bypass_named_maps
    @table = create_table(user_id: @user.id)
  end

  after(:each) do
    @user.destroy
  end

  after(:all) do
    bypass_named_maps
    @user.destroy
  end

  it 'raises an 8004 error when merging tables
  through columns with different types' do
    table1 = create_table(user_id: @user.id)
    table2 = create_table(user_id: @user.id)

    table1.modify_column!(name: 'name', type: 'double precision')
    table1.insert_row!(name: 1.0)
    table2.insert_row!(name: '1')

    merge_query = %(
      SELECT #{table2.name}.the_geom,
              #{table2.name}.description,
              #{table2.name}.name,
              #{table1.name}.the_geom AS #{table1.name}_the_geom,
              #{table1.name}.description AS #{table1.name}_description,
              #{table1.name}.name AS #{table1.name}_name
      FROM #{table2.name} FULL OUTER JOIN #{table1.name}
      ON #{table2.name}.name = #{table1.name}.name
    )
    data_import = DataImport.create(
      user_id:  @user.id,
      table_name: "merged_table",
      from_query: merge_query
    )
    data_import.run_import!
    data_import.error_code.should == 8004
  end

  it 'raises a meaningful error if cartodb_id is not valid' do
    Table.any_instance.stubs(:cartodbfy).raises(CartoDB::CartoDBfyInvalidID)
    data_import = DataImport.create(
      user_id: @user.id,
      data_source: fake_data_path('clubbing.csv'),
      updated_at: Time.now
    )

    data_import.run_import!
    data_import.error_code.should == 2011
  end

  it 'should overwrite dataset if collision_strategy is set to overwrite' do
    carto_user = Carto::User.find(@user.id)
    carto_user.visualizations.count.should eq 1

    data_import = create_import(overwrite: false, truncated: false)
    data_import.run_import!
    carto_user.reload
    carto_user.visualizations.count.should eq 2
    data_import.state.should eq 'complete'
    data_import.table_name.should eq 'walmart_latlon'
    data_import.user.in_database["select count(*) from #{data_import.table_name}"].all[0][:count].should eq 3176

    data_import = create_import(overwrite: false, truncated: false)
    data_import.run_import!
    carto_user.reload
    carto_user.visualizations.count.should eq 3
    data_import.state.should eq 'complete'
    data_import.table_name.should eq 'walmart_latlon_1'
    data_import.user.in_database["select count(*) from #{data_import.table_name}"].all[0][:count].should eq 3176

    data_import = create_import(overwrite: true, truncated: true)
    data_import.run_import!
    carto_user.reload
    carto_user.visualizations.count.should eq 3
    data_import.state.should eq 'complete'
    data_import.table_name.should eq 'walmart_latlon'
    data_import.user.in_database["select count(*) from #{data_import.table_name}"].all[0][:count].should eq 2
  end

  it 'should raise an exception if overwriting with missing data' do
    carto_user = Carto::User.find(@user.id)
    carto_user.visualizations.count.should eq 1

    data_import = create_import(overwrite: false, truncated: true)
    data_import.run_import!
    carto_user.reload
    carto_user.visualizations.count.should eq 2
    data_import.state.should eq 'complete'
    data_import.table_name.should eq 'walmart_latlon'
    data_import.user.in_database["select count(*) from #{data_import.table_name}"].all[0][:count].should eq 2

    data_import = create_import(overwrite: true, truncated: true, incomplete_schema: true)
    expect { data_import.run_import! }.to raise_error(RuntimeError, 'Incompatible schemas')
    data_import.log.entries.should match(/Exception: Incompatible schemas/)
  end

  it 'should not raise exceptions if overwriting with more data' do
    carto_user = Carto::User.find(@user.id)
    carto_user.visualizations.count.should eq 1

    data_import = create_import(overwrite: false, truncated: true, incomplete_schema: true)
    data_import.run_import!
    carto_user.reload
    carto_user.visualizations.count.should eq 2
    data_import.state.should eq 'complete'
    data_import.table_name.should eq 'walmart_latlon'
    data_import.user.in_database["select count(*) from #{data_import.table_name}"].all[0][:count].should eq 2

    data_import = create_import(overwrite: true, truncated: true, incomplete_schema: false)
    data_import.run_import!
    carto_user.reload
    carto_user.visualizations.count.should eq 2
    data_import.state.should eq 'complete'
    data_import.table_name.should eq 'walmart_latlon'
    data_import.user.in_database["select count(*) from #{data_import.table_name}"].all[0][:count].should eq 2
  end

  def create_import(overwrite:, truncated:, incomplete_schema: false)<commit_msg>Add validation that checks all tables are properly registered (including cartodbfiation)<commit_after># encoding: utf-8
require_relative '../spec_helper'
require_relative 'data_import_shared_examples'

describe DataImport do
  let(:data_import_class) { ::DataImport }
  it_behaves_like 'DataImport model'

  before(:each) do
    @user = create_user
    bypass_named_maps
    @table = create_table(user_id: @user.id)
  end

  after(:each) do
    @user.destroy
  end

  after(:all) do
    bypass_named_maps
    @user.destroy
  end

  it 'raises an 8004 error when merging tables
  through columns with different types' do
    table1 = create_table(user_id: @user.id)
    table2 = create_table(user_id: @user.id)

    table1.modify_column!(name: 'name', type: 'double precision')
    table1.insert_row!(name: 1.0)
    table2.insert_row!(name: '1')

    merge_query = %(
      SELECT #{table2.name}.the_geom,
              #{table2.name}.description,
              #{table2.name}.name,
              #{table1.name}.the_geom AS #{table1.name}_the_geom,
              #{table1.name}.description AS #{table1.name}_description,
              #{table1.name}.name AS #{table1.name}_name
      FROM #{table2.name} FULL OUTER JOIN #{table1.name}
      ON #{table2.name}.name = #{table1.name}.name
    )
    data_import = DataImport.create(
      user_id:  @user.id,
      table_name: "merged_table",
      from_query: merge_query
    )
    data_import.run_import!
    data_import.error_code.should == 8004
  end

  it 'raises a meaningful error if cartodb_id is not valid' do
    Table.any_instance.stubs(:cartodbfy).raises(CartoDB::CartoDBfyInvalidID)
    data_import = DataImport.create(
      user_id: @user.id,
      data_source: fake_data_path('clubbing.csv'),
      updated_at: Time.now
    )

    data_import.run_import!
    data_import.error_code.should == 2011
  end

  it 'should overwrite dataset if collision_strategy is set to overwrite' do
    carto_user = Carto::User.find(@user.id)
    carto_user.visualizations.count.should eq 1

    data_import = create_import(overwrite: false, truncated: false)
    data_import.run_import!
    carto_user.reload
    carto_user.visualizations.count.should eq 2
    data_import.state.should eq 'complete'
    data_import.table_name.should eq 'walmart_latlon'
    data_import.user.in_database["select count(*) from #{data_import.table_name}"].all[0][:count].should eq 3176
    user_tables_should_be_registered

    data_import = create_import(overwrite: false, truncated: false)
    data_import.run_import!
    carto_user.reload
    carto_user.visualizations.count.should eq 3
    data_import.state.should eq 'complete'
    data_import.table_name.should eq 'walmart_latlon_1'
    data_import.user.in_database["select count(*) from #{data_import.table_name}"].all[0][:count].should eq 3176
    user_tables_should_be_registered

    data_import = create_import(overwrite: true, truncated: true)
    data_import.run_import!
    carto_user.reload
    carto_user.visualizations.count.should eq 3
    data_import.state.should eq 'complete'
    data_import.table_name.should eq 'walmart_latlon'
    data_import.user.in_database["select count(*) from #{data_import.table_name}"].all[0][:count].should eq 2
    user_tables_should_be_registered
  end

  it 'should raise an exception if overwriting with missing data' do
    carto_user = Carto::User.find(@user.id)
    carto_user.visualizations.count.should eq 1

    data_import = create_import(overwrite: false, truncated: true)
    data_import.run_import!
    carto_user.reload
    carto_user.visualizations.count.should eq 2
    data_import.state.should eq 'complete'
    data_import.table_name.should eq 'walmart_latlon'
    data_import.user.in_database["select count(*) from #{data_import.table_name}"].all[0][:count].should eq 2
    user_tables_should_be_registered

    data_import = create_import(overwrite: true, truncated: true, incomplete_schema: true)
    expect { data_import.run_import! }.to raise_error(RuntimeError, 'Incompatible schemas')
    data_import.log.entries.should match(/Exception: Incompatible schemas/)
  end

  it 'should not raise exceptions if overwriting with more data' do
    carto_user = Carto::User.find(@user.id)
    carto_user.visualizations.count.should eq 1

    data_import = create_import(overwrite: false, truncated: true, incomplete_schema: true)
    data_import.run_import!
    carto_user.reload
    carto_user.visualizations.count.should eq 2
    data_import.state.should eq 'complete'
    data_import.table_name.should eq 'walmart_latlon'
    data_import.user.in_database["select count(*) from #{data_import.table_name}"].all[0][:count].should eq 2
    user_tables_should_be_registered

    data_import = create_import(overwrite: true, truncated: true, incomplete_schema: false)
    data_import.run_import!
    carto_user.reload
    carto_user.visualizations.count.should eq 2
    data_import.state.should eq 'complete'
    data_import.table_name.should eq 'walmart_latlon'
    data_import.user.in_database["select count(*) from #{data_import.table_name}"].all[0][:count].should eq 2
    user_tables_should_be_registered
  end

  def user_tables_should_be_registered
    Carto::GhostTablesManager.new(@user.id).user_tables_synced_with_db?.should eq true
  end

  def create_import(overwrite:, truncated:, incomplete_schema: false)
{'fraction_non_alphanumeric': 0.10268378063010501, 'fraction_numerical': 0.012057565149747181, 'mean_word_length': 4.051571709233792, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4530835', 'n_tokens_mistral': 3487, 'n_tokens_neox': 3326, 'n_words': 681}