<commit_before><?php
namespace Keboola\Json;

use Keboola\Json\Test\ParserTestCase;

class RealDataTest extends ParserTestCase
{
    public function testProcess()
    {
        $parser = $this->getParser();

        $testFilesPath = $this->getDataDir() . 'Json_tweets_pinkbike';

        $data = $this->loadJson('Json_tweets_pinkbike');

        $parser->process($data);

        foreach ($parser->getCsvFiles() as $name => $table) {
            // compare result files
            self::assertEquals(
                file_get_contents("{$testFilesPath}/{$name}.csv"),
                file_get_contents($table->getPathname())
            );

            // compare column counts
            $parsedFile = file($table->getPathname());
            foreach ($parsedFile as $row) {
                if (empty($headerCount)) {
                    $headerCount = count($row);
                } else {
                    self::assertEquals($headerCount, count($row));
                }
            }
        }

        // make sure all the files are present
        $dir = scandir($testFilesPath);
        array_walk($dir, function (&$val) {
            $val = str_replace(".csv", "", $val);
        });
        self::assertEquals([".",".."], array_diff($dir, array_keys($parser->getCsvFiles())));
        self::assertContainsOnlyInstancesOf('\Keboola\CsvTable\Table', $parser->getCsvFiles());
    }

    public function testTypeCharacters()
    {
        $parser = $this->getParser();

        $data = $this->loadJson('Json_tweets_pinkbike');

        $parser->process($data, 'a/b.c&d@e$f');

        self::assertEquals(
            [
                'a_b_c_d_e_f',
                'a_b_c_d_e_f_statuses',
                'a_b_c_d_e_f_statuses_user_entities_url_urls',
                'a_b_c_d_e_f_statuses_user_entities_url_urls_indices',
                'a_b_c_d_e_f_statuses_entities_hashtags',
                'a_b_c_d_e_f_statuses_entities_hashtags_indices',
                'a_b_c_d_e_f_statuses_entities_urls',
                'a_b_c_d_e_f_statuses_entities_urls_indices',
                'a_b_c_d_e_f_statuses_entities_user_mentions',
                'a_b_c_d_e_f_statuses_entities_user_mentions_indices',
                'a_b_c_d_e_f_statuses_retweeted_status_user_entities_url_urls',
                'abcdefsrueuui__status_user_entities_url_urls_indices',
                'a_b_c_d_e_f_statuses_retweeted_status_entities_urls',
                'a_b_c_d_e_f_statuses_retweeted_status_entities_urls_indices',
                'a_b_c_d_e_f_statuses_retweeted_status_entities_user_mentions',
                'abcdefsreui__status_entities_user_mentions_indices',
                'a_b_c_d_e_f_statuses_user_entities_description_urls',
                'a_b_c_d_e_f_statuses_user_entities_description_urls_indices',
                'a_b_c_d_e_f_statuses_entities_media',
                'a_b_c_d_e_f_statuses_entities_media_indices',
                'a_b_c_d_e_f_statuses_retweeted_status_entities_media',
                'a_b_c_d_e_f_statuses_retweeted_status_entities_media_indices',
            ],
            array_keys($parser->getCsvFiles())
        );
    }

    public function testRowCount()
    {
        $parser = $this->getParser();

        $data = $this->loadJson('Json_tweets_pinkbike');

        $parser->process($data);<commit_msg>behavior change: more tests for column names shortening<commit_after><?php
namespace Keboola\Json;

use Keboola\Json\Test\ParserTestCase;

class RealDataTest extends ParserTestCase
{
    public function testProcess()
    {
        $parser = $this->getParser();

        $testFilesPath = $this->getDataDir() . 'Json_tweets_pinkbike';

        $data = $this->loadJson('Json_tweets_pinkbike');

        $parser->process($data);

        foreach ($parser->getCsvFiles() as $name => $table) {
            // compare result files
            self::assertEquals(
                file_get_contents("{$testFilesPath}/{$name}.csv"),
                file_get_contents($table->getPathname())
            );

            // compare column counts
            $parsedFile = file($table->getPathname());
            foreach ($parsedFile as $row) {
                if (empty($headerCount)) {
                    $headerCount = count($row);
                } else {
                    self::assertEquals($headerCount, count($row));
                }
            }
        }

        // make sure all the files are present
        $dir = scandir($testFilesPath);
        array_walk($dir, function (&$val) {
            $val = str_replace(".csv", "", $val);
        });
        self::assertEquals([".",".."], array_diff($dir, array_keys($parser->getCsvFiles())));
        self::assertContainsOnlyInstancesOf('\Keboola\CsvTable\Table', $parser->getCsvFiles());
    }

    public function testTypeCharacters()
    {
        $parser = $this->getParser();

        $data = $this->loadJson('Json_tweets_pinkbike');

        $parser->process($data, 'a/b.c&d@e$f');

        self::assertEquals(
            [
                'a_b_c_d_e_f',
                'a_b_c_d_e_f_statuses',
                'a_b_c_d_e_f_statuses_user_entities_url_urls',
                'a_b_c_d_e_f_statuses_user_entities_url_urls_indices',
                'a_b_c_d_e_f_statuses_entities_hashtags',
                'a_b_c_d_e_f_statuses_entities_hashtags_indices',
                'a_b_c_d_e_f_statuses_entities_urls',
                'a_b_c_d_e_f_statuses_entities_urls_indices',
                'a_b_c_d_e_f_statuses_entities_user_mentions',
                'a_b_c_d_e_f_statuses_entities_user_mentions_indices',
                'a_b_c_d_e_f_statuses_retweeted_status_user_entities_url_urls',
                'a_e_f_statuses_retweeted_status_user_entities_url_urls_indices',
                'a_b_c_d_e_f_statuses_retweeted_status_entities_urls',
                'a_b_c_d_e_f_statuses_retweeted_status_entities_urls_indices',
                'a_b_c_d_e_f_statuses_retweeted_status_entities_user_mentions',
                'a_e_f_statuses_retweeted_status_entities_user_mentions_indices',
                'a_b_c_d_e_f_statuses_user_entities_description_urls',
                'a_b_c_d_e_f_statuses_user_entities_description_urls_indices',
                'a_b_c_d_e_f_statuses_entities_media',
                'a_b_c_d_e_f_statuses_entities_media_indices',
                'a_b_c_d_e_f_statuses_retweeted_status_entities_media',
                'a_b_c_d_e_f_statuses_retweeted_status_entities_media_indices',
            ],
            array_keys($parser->getCsvFiles())
        );
    }

    public function testRowCount()
    {
        $parser = $this->getParser();

        $data = $this->loadJson('Json_tweets_pinkbike');

        $parser->process($data);
{'fraction_non_alphanumeric': 0.10566093657379964, 'fraction_numerical': 0.0, 'mean_word_length': 2.384653961885657, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2474166', 'n_tokens_mistral': 2282, 'n_tokens_neox': 2263, 'n_words': 222}