<commit_before><?php

namespace Keboola\Test\Backend\Workspaces;

use Keboola\StorageApi\ClientException;
use Keboola\StorageApi\Workspaces;
use Keboola\Csv\CsvFile;
use Keboola\Test\Backend\Workspaces\Backend\WorkspaceBackendFactory;

class LegacyWorkspacesRedshiftTest extends ParallelWorkspacesTestCase
{

    /**
     * @dataProvider columnCompressionDataTypesDefinitions
     * @param $dataTypesDefinition
     */
    public function testColumnCompression($dataTypesDefinition)
    {
        $workspaces = new Workspaces($this->workspaceSapiClient);
        $workspace = $this->initTestWorkspace();

        // Create a table of sample data
        $importFile = __DIR__ . '/../../_data/languages.csv';
        $tableId = $this->_client->createTable(
            $this->getTestBucketId(self::STAGE_IN),
            'languages-rs',
            new CsvFile($importFile)
        );

        $workspaces->loadWorkspaceData($workspace['id'], [
            "input" => [
                [
                    "source" => $tableId,
                    "destination" => "languages-rs",
                    "datatypes" => $dataTypesDefinition
                ]
            ]
        ]);

        $backend = WorkspaceBackendFactory::createWorkspaceBackend($workspace);
        $table = $backend->describeTableColumns('languages-rs');

        $this->assertEquals("varchar", $table['id']['DATA_TYPE']);
        $this->assertEquals(50, $table['id']['LENGTH']);
        $this->assertEquals("lzo", $table['id']['COMPRESSION']);

        $this->assertEquals("varchar", $table['name']['DATA_TYPE']);
        $this->assertEquals(255, $table['name']['LENGTH']);
        $this->assertEquals("bytedict", $table['name']['COMPRESSION']);
    }

    public function columnCompressionDataTypesDefinitions()
    {
        return [
            [
                [
                    'id' => "VARCHAR(50)",
                    "name" => "VARCHAR(255) ENCODE BYTEDICT"
                ]
            ],
            [
                [
                    [
                        'column' => 'id',
                        'type' => 'VARCHAR',
                        'length' => '50'
                    ],
                    [
                        'column' => 'name',
                        'type' => 'VARCHAR',
                        'length' => '255',
                        'compression' => 'BYTEDICT'
                    ]
                ]
            ]
        ];
    }

    public function testLoadedPrimaryKeys()
    {
        $primaries = ['Paid_Search_Engine_Account', 'Date', 'Paid_Search_Campaign', 'Paid_Search_Ad_ID', 'Site__DFA'];
        $pkTableId = $this->_client->createTable(
            $this->getTestBucketId(self::STAGE_IN),
            'languages-pk',
            new CsvFile(__DIR__ . '/../../_data/multiple-columns-pk.csv'),
            array(
                'primaryKey' => implode(",", $primaries),
            )
        );

        $mapping = [
            "source" => $pkTableId,
            "destination" => "languages-pk"
        ];

        $workspaces = new Workspaces($this->workspaceSapiClient);
        $workspace = $this->initTestWorkspace();
        $backend = WorkspaceBackendFactory::createWorkspaceBackend($workspace);

        $workspaces->loadWorkspaceData($workspace['id'], ["input" => [$mapping]]);

        $cols = $backend->describeTableColumns("languages-pk");
        $this->assertCount(6, $cols);
        $this->assertEquals("varchar", $cols['paid_search_engine_account']['DATA_TYPE']);
        $this->assertEquals(255, $cols['paid_search_engine_account']['LENGTH']);
        $this->assertEquals("varchar", $cols['date']['DATA_TYPE']);
        $this->assertEquals(255, $cols['date']['LENGTH']);
        $this->assertEquals("varchar", $cols['paid_search_campaign']['DATA_TYPE']);
        $this->assertEquals(255, $cols['paid_search_campaign']['LENGTH']);
        $this->assertEquals("varchar", $cols['paid_search_ad_id']['DATA_TYPE']);
        $this->assertEquals(255, $cols['paid_search_ad_id']['LENGTH']);
        $this->assertEquals("varchar", $cols['site__dfa']['DATA_TYPE']);
        $this->assertEquals(255, $cols['site__dfa']['LENGTH']);
        $this->assertEquals("varchar", $cols['advertiser_id']['DATA_TYPE']);
        $this->assertEquals(65535, $cols['advertiser_id']['LENGTH']);

        // Check that PK is NOT set if not all PK columns are present
        $mapping2 = [
            "source" => $pkTableId,
            "destination" => "languages-pk-skipped",
            "columns" => ['Paid_Search_Engine_Account', 'Date'] // missing PK columns
        ];
        $workspaces->loadWorkspaceData($workspace['id'], ["input" => [$mapping2]]);

        $cols = $backend->describeTableColumns("languages-pk-skipped");
        $this->assertCount(2, $cols);
        $this->assertEquals("varchar", $cols['paid_search_engine_account']['DATA_TYPE']);
        $this->assertEquals(255, $cols['paid_search_engine_account']['LENGTH']);
        $this->assertEquals("varchar", $cols['date']['DATA_TYPE']);
        $this->assertEquals(255, $cols['date']['LENGTH']);
    }

    public function testLoadIncrementalNotNullable()
    {
        $bucketId = $this->getTestBucketId(self::STAGE_IN);

        $workspaces = new Workspaces($this->workspaceSapiClient);
        $workspace = $this->initTestWorkspace();
        $backend = WorkspaceBackendFactory::createWorkspaceBackend($workspace);


        $importFile = __DIR__ . '/../../_data/languages.with-state.csv';
        $tableId = $this->_client->createTable(
            $bucketId,
            'languages',
            new CsvFile($importFile),
            ['primaryKey' => 'id']
        );

        // first load
        $options = [
            'input' => [
                [
                    'source' => $tableId,
                    'destination' => 'languages',<commit_msg>expect to PK on RS backend to have length 65535<commit_after><?php

namespace Keboola\Test\Backend\Workspaces;

use Keboola\StorageApi\ClientException;
use Keboola\StorageApi\Workspaces;
use Keboola\Csv\CsvFile;
use Keboola\Test\Backend\Workspaces\Backend\WorkspaceBackendFactory;

class LegacyWorkspacesRedshiftTest extends ParallelWorkspacesTestCase
{

    /**
     * @dataProvider columnCompressionDataTypesDefinitions
     * @param $dataTypesDefinition
     */
    public function testColumnCompression($dataTypesDefinition)
    {
        $workspaces = new Workspaces($this->workspaceSapiClient);
        $workspace = $this->initTestWorkspace();

        // Create a table of sample data
        $importFile = __DIR__ . '/../../_data/languages.csv';
        $tableId = $this->_client->createTable(
            $this->getTestBucketId(self::STAGE_IN),
            'languages-rs',
            new CsvFile($importFile)
        );

        $workspaces->loadWorkspaceData($workspace['id'], [
            "input" => [
                [
                    "source" => $tableId,
                    "destination" => "languages-rs",
                    "datatypes" => $dataTypesDefinition
                ]
            ]
        ]);

        $backend = WorkspaceBackendFactory::createWorkspaceBackend($workspace);
        $table = $backend->describeTableColumns('languages-rs');

        $this->assertEquals("varchar", $table['id']['DATA_TYPE']);
        $this->assertEquals(50, $table['id']['LENGTH']);
        $this->assertEquals("lzo", $table['id']['COMPRESSION']);

        $this->assertEquals("varchar", $table['name']['DATA_TYPE']);
        $this->assertEquals(255, $table['name']['LENGTH']);
        $this->assertEquals("bytedict", $table['name']['COMPRESSION']);
    }

    public function columnCompressionDataTypesDefinitions()
    {
        return [
            [
                [
                    'id' => "VARCHAR(50)",
                    "name" => "VARCHAR(255) ENCODE BYTEDICT"
                ]
            ],
            [
                [
                    [
                        'column' => 'id',
                        'type' => 'VARCHAR',
                        'length' => '50'
                    ],
                    [
                        'column' => 'name',
                        'type' => 'VARCHAR',
                        'length' => '255',
                        'compression' => 'BYTEDICT'
                    ]
                ]
            ]
        ];
    }

    public function testLoadedPrimaryKeys()
    {
        $primaries = ['Paid_Search_Engine_Account', 'Date', 'Paid_Search_Campaign', 'Paid_Search_Ad_ID', 'Site__DFA'];
        $pkTableId = $this->_client->createTable(
            $this->getTestBucketId(self::STAGE_IN),
            'languages-pk',
            new CsvFile(__DIR__ . '/../../_data/multiple-columns-pk.csv'),
            array(
                'primaryKey' => implode(",", $primaries),
            )
        );

        $mapping = [
            "source" => $pkTableId,
            "destination" => "languages-pk"
        ];

        $workspaces = new Workspaces($this->workspaceSapiClient);
        $workspace = $this->initTestWorkspace();
        $backend = WorkspaceBackendFactory::createWorkspaceBackend($workspace);

        $workspaces->loadWorkspaceData($workspace['id'], ["input" => [$mapping]]);

        $cols = $backend->describeTableColumns("languages-pk");
        $this->assertCount(6, $cols);
        $this->assertEquals("varchar", $cols['paid_search_engine_account']['DATA_TYPE']);
        $this->assertEquals(65535, $cols['paid_search_engine_account']['LENGTH']);
        $this->assertEquals("varchar", $cols['date']['DATA_TYPE']);
        $this->assertEquals(65535, $cols['date']['LENGTH']);
        $this->assertEquals("varchar", $cols['paid_search_campaign']['DATA_TYPE']);
        $this->assertEquals(65535, $cols['paid_search_campaign']['LENGTH']);
        $this->assertEquals("varchar", $cols['paid_search_ad_id']['DATA_TYPE']);
        $this->assertEquals(65535, $cols['paid_search_ad_id']['LENGTH']);
        $this->assertEquals("varchar", $cols['site__dfa']['DATA_TYPE']);
        $this->assertEquals(65535, $cols['site__dfa']['LENGTH']);
        $this->assertEquals("varchar", $cols['advertiser_id']['DATA_TYPE']);
        $this->assertEquals(65535, $cols['advertiser_id']['LENGTH']);

        // Check that PK is NOT set if not all PK columns are present
        $mapping2 = [
            "source" => $pkTableId,
            "destination" => "languages-pk-skipped",
            "columns" => ['Paid_Search_Engine_Account', 'Date'] // missing PK columns
        ];
        $workspaces->loadWorkspaceData($workspace['id'], ["input" => [$mapping2]]);

        $cols = $backend->describeTableColumns("languages-pk-skipped");
        $this->assertCount(2, $cols);
        $this->assertEquals("varchar", $cols['paid_search_engine_account']['DATA_TYPE']);
        $this->assertEquals(65535, $cols['paid_search_engine_account']['LENGTH']);
        $this->assertEquals("varchar", $cols['date']['DATA_TYPE']);
        $this->assertEquals(65535, $cols['date']['LENGTH']);
    }

    public function testLoadIncrementalNotNullable()
    {
        $bucketId = $this->getTestBucketId(self::STAGE_IN);

        $workspaces = new Workspaces($this->workspaceSapiClient);
        $workspace = $this->initTestWorkspace();
        $backend = WorkspaceBackendFactory::createWorkspaceBackend($workspace);


        $importFile = __DIR__ . '/../../_data/languages.with-state.csv';
        $tableId = $this->_client->createTable(
            $bucketId,
            'languages',
            new CsvFile($importFile),
            ['primaryKey' => 'id']
        );

        // first load
        $options = [
            'input' => [
                [
                    'source' => $tableId,
                    'destination' => 'languages',
{'fraction_non_alphanumeric': 0.1626209472076048, 'fraction_numerical': 0.009251400441351214, 'mean_word_length': 2.2621816168327795, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 143, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6077275', 'n_tokens_mistral': 3382, 'n_tokens_neox': 3245, 'n_words': 454}