<commit_before><?php
/**
 * Created by JetBrains PhpStorm.
 * User: martinhalamicek
 * Date: 22/05/14
 * Time: 16:38
 * To change this template use File | Settings | File Templates.
 */

namespace Keboola\Test\Backend\CommonPart1;

use Keboola\StorageApi\ClientException;
use Keboola\StorageApi\TableExporter;
use Keboola\Test\StorageApiTestCase;
use Keboola\StorageApi\Client;
use Keboola\Csv\CsvFile;

class DataPreviewLimitsTest extends StorageApiTestCase
{

    public function setUp()
    {
        parent::setUp();
        $this->_initEmptyTestBuckets();
    }


    public function testDataPreviewDefaultLimit()
    {
        $csvFile = $this->generateCsv(2000);
        $tableId = $this->_client->createTable($this->getTestBucketId(), 'users', $csvFile);

        $preview = $this->_client->getTableDataPreview($tableId);
        $this->assertCount(100, Client::parseCsv($preview), 'only preview of 100 rows should be returned');

        $tableExporter = new TableExporter($this->_client);

        $fullTableExportPath = tempnam(sys_get_temp_dir(), 'keboola');
        $tableExporter->exportTable($tableId, $fullTableExportPath, []);
        $this->assertCount(2000, Client::parseCsv(file_get_contents($fullTableExportPath)));
    }

    public function testDataPreviewParametrizedLimit()
    {
        $csvFile = $this->generateCsv(2000);
        $tableId = $this->_client->createTable($this->getTestBucketId(), 'users', $csvFile);

        $preview = $this->_client->getTableDataPreview($tableId, [
            'limit' => 2,
        ]);
        $this->assertCount(2, Client::parseCsv($preview), 'only preview of 2 rows should be returned');
    }

    public function testDataPreviewMaximumLimit()
    {
        $csvFile = $this->generateCsv(2000);
        $tableId = $this->_client->createTable($this->getTestBucketId(), 'users', $csvFile);

        try {
            $this->_client->getTableDataPreview($tableId, [
                'limit' => 1200,
            ]);
            $this->fail('limit 1200 should not be allowed');
        } catch (ClientException $e) {
            $this->assertEquals(400, $e->getCode());
            $this->assertEquals('storage.tables.validation', $e->getStringCode());
            $this->assertContains('1000', $e->getMessage());
        }
    }


    public function testJsonTruncationLimit()
    {
        $columnCount = 5;
        $rowCount = 5;
        $csvFile = $this->generateCsv($rowCount - 1, $columnCount);
        $row = [];
        for ($i = 0; $i < $columnCount; $i++) {
            $row[] = $this->createRandomString(20000);
        }
        $csvFile->writeRow($row);
        $tableId = $this->_client->createTable($this->getTestBucketId(), 'slim', $csvFile);
        $jsonPreview = $this->_client->getTableDataPreview($tableId, ['format' => 'json']);
        $this->assertSame($columnCount, count($jsonPreview['columns']));
        $this->assertSame($rowCount, count($jsonPreview['rows']));
        $this->assertContains('col' . ($columnCount - 1), $jsonPreview['columns']);
        $truncatedRow = $this->getTruncatedRow($jsonPreview);
        $this->assertNotEmpty($truncatedRow);
        $this->assertGreaterThan(16000, mb_strlen($truncatedRow[0]['value']), 'Value in row is not truncated');
        $this->assertLessThan(19999, mb_strlen($truncatedRow[0]['value']), 'Value in row is not truncated');
    }

    private function getTruncatedRow(array $jsonPreview)
    {
        foreach ($jsonPreview['rows'] as $row) {
            if ($row[0]['isTruncated'] === "1") {
                return $row;
            }
        }
        return [];
    }


    private function generateCsv(int $rowsCount, int $collsCount = 2)<commit_msg>Comparing truncated collumn to exact length<commit_after><?php
/**
 * Created by JetBrains PhpStorm.
 * User: martinhalamicek
 * Date: 22/05/14
 * Time: 16:38
 * To change this template use File | Settings | File Templates.
 */

namespace Keboola\Test\Backend\CommonPart1;

use Keboola\StorageApi\ClientException;
use Keboola\StorageApi\TableExporter;
use Keboola\Test\StorageApiTestCase;
use Keboola\StorageApi\Client;
use Keboola\Csv\CsvFile;

class DataPreviewLimitsTest extends StorageApiTestCase
{

    public function setUp()
    {
        parent::setUp();
        $this->_initEmptyTestBuckets();
    }


    public function testDataPreviewDefaultLimit()
    {
        $csvFile = $this->generateCsv(2000);
        $tableId = $this->_client->createTable($this->getTestBucketId(), 'users', $csvFile);

        $preview = $this->_client->getTableDataPreview($tableId);
        $this->assertCount(100, Client::parseCsv($preview), 'only preview of 100 rows should be returned');

        $tableExporter = new TableExporter($this->_client);

        $fullTableExportPath = tempnam(sys_get_temp_dir(), 'keboola');
        $tableExporter->exportTable($tableId, $fullTableExportPath, []);
        $this->assertCount(2000, Client::parseCsv(file_get_contents($fullTableExportPath)));
    }

    public function testDataPreviewParametrizedLimit()
    {
        $csvFile = $this->generateCsv(2000);
        $tableId = $this->_client->createTable($this->getTestBucketId(), 'users', $csvFile);

        $preview = $this->_client->getTableDataPreview($tableId, [
            'limit' => 2,
        ]);
        $this->assertCount(2, Client::parseCsv($preview), 'only preview of 2 rows should be returned');
    }

    public function testDataPreviewMaximumLimit()
    {
        $csvFile = $this->generateCsv(2000);
        $tableId = $this->_client->createTable($this->getTestBucketId(), 'users', $csvFile);

        try {
            $this->_client->getTableDataPreview($tableId, [
                'limit' => 1200,
            ]);
            $this->fail('limit 1200 should not be allowed');
        } catch (ClientException $e) {
            $this->assertEquals(400, $e->getCode());
            $this->assertEquals('storage.tables.validation', $e->getStringCode());
            $this->assertContains('1000', $e->getMessage());
        }
    }


    public function testJsonTruncationLimit()
    {
        $columnCount = 5;
        $rowCount = 5;
        $csvFile = $this->generateCsv($rowCount - 1, $columnCount);
        $row = [];
        for ($i = 0; $i < $columnCount; $i++) {
            $row[] = $this->createRandomString(20000);
        }
        $csvFile->writeRow($row);
        $tableId = $this->_client->createTable($this->getTestBucketId(), 'slim', $csvFile);
        $jsonPreview = $this->_client->getTableDataPreview($tableId, ['format' => 'json']);
        $this->assertSame($columnCount, count($jsonPreview['columns']));
        $this->assertSame($rowCount, count($jsonPreview['rows']));
        $this->assertContains('col' . ($columnCount - 1), $jsonPreview['columns']);
        $truncatedRow = $this->getTruncatedRow($jsonPreview);
        $this->assertNotEmpty($truncatedRow);
        $this->assertEquals(16383, mb_strlen($truncatedRow[0]['value']), 'Value in row is not truncated');
    }

    private function getTruncatedRow(array $jsonPreview)
    {
        foreach ($jsonPreview['rows'] as $row) {
            if ($row[0]['isTruncated'] === "1") {
                return $row;
            }
        }
        return [];
    }


    private function generateCsv(int $rowsCount, int $collsCount = 2)
{'fraction_non_alphanumeric': 0.15535176567205036, 'fraction_numerical': 0.019983575143717493, 'mean_word_length': 3.0844046953605364, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 100, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7967', 'n_tokens_mistral': 2304, 'n_tokens_neox': 2155, 'n_words': 396}