<commit_before><?php

namespace MLocati\IDNA\Tests;

use PHPUnit_Framework_TestCase;
use MLocati\IDNA\IdnaMap;
use MLocati\IDNA\CodepointConverter\CodepointConverterInterface;

class IdnaMapTest extends PHPUnit_Framework_TestCase
{
    public function allCodepointsAreCoveredProvider()
    {
        return [
            [CodepointConverterInterface::MIN_CODEPOINT, CodepointConverterInterface::MAX_CODEPOINT, true],
            [CodepointConverterInterface::MIN_CODEPOINT, CodepointConverterInterface::MAX_CODEPOINT, false],
        ];
    }
    /**
     * @dataProvider allCodepointsAreCoveredProvider
     */
    public function testAllCodepointsAreCoveredProvider($from, $to, $useSTD3ASCIIRules)
    {
        if (!class_exists(IdnaMap::class, true)) {
            $this->markTestSkipped('The IdnaMap can\'t be found (you may need to create it with the create-idnamap command)');
        }
        if (!is_callable([IdnaMap::class, 'isDisallowed'])) {
            $this->markTestSkipped('The IdnaMap must be created with Disallowed support (--debug option of the create-idnamap command)');
        }
        for ($codepoint = $from; $codepoint <= $to; ++$codepoint) {
            $covered = false;
            if (IdnaMap::isDisallowed($codepoint, $useSTD3ASCIIRules)) {
                $covered = true;
            } elseif (IdnaMap::getMapped($codepoint, $useSTD3ASCIIRules) !== null) {
                $covered = true;
            } elseif (IdnaMap::getDeviation($codepoint) !== null) {
                $covered = true;
            } elseif (IdnaMap::isIgnored($codepoint)) {
                $covered = true;
            } elseif (IdnaMap::isValid($codepoint, [], $useSTD3ASCIIRules)) {
                $covered = true;
            }
            $this->assertTrue($covered, "Check that code point $codepoint (0x".dechex($codepoint).') is covered');
        }
    }
}
<commit_msg>Split IdnaMap to show some progress<commit_after><?php

namespace MLocati\IDNA\Tests;

use PHPUnit_Framework_TestCase;
use MLocati\IDNA\IdnaMap;
use MLocati\IDNA\CodepointConverter\CodepointConverterInterface;

class IdnaMapTest extends PHPUnit_Framework_TestCase
{
    public function allCodepointsAreCoveredProvider()
    {
        $data = [];
        $min = CodepointConverterInterface::MIN_CODEPOINT;
        $max = CodepointConverterInterface::MAX_CODEPOINT;
        $perStep = (int) max(10, ($max - $min + 1) / 1000);
        foreach ([true, false] as $useSTD3ASCIIRules) {
            for ($start = $min; $start <= $max; $start += $perStep) {
                $end = min($max, $start + $perStep);
                $data[] = [$start, $end, $useSTD3ASCIIRules];
            }
        }

        return $data;
    }

    /**
     * @dataProvider allCodepointsAreCoveredProvider
     */
    public function testAllCodepointsAreCoveredProvider($from, $to, $useSTD3ASCIIRules)
    {
        if (!class_exists(IdnaMap::class, true)) {
            $this->markTestSkipped('The IdnaMap can\'t be found (you may need to create it with the create-idnamap command)');
        }
        if (!is_callable([IdnaMap::class, 'isDisallowed'])) {
            $this->markTestSkipped('The IdnaMap must be created with Disallowed support (--debug option of the create-idnamap command)');
        }
        for ($codepoint = $from; $codepoint <= $to; ++$codepoint) {
            $covered = false;
            if (IdnaMap::isDisallowed($codepoint, $useSTD3ASCIIRules)) {
                $covered = true;
            } elseif (IdnaMap::getMapped($codepoint, $useSTD3ASCIIRules) !== null) {
                $covered = true;
            } elseif (IdnaMap::getDeviation($codepoint) !== null) {
                $covered = true;
            } elseif (IdnaMap::isIgnored($codepoint)) {
                $covered = true;
            } elseif (IdnaMap::isValid($codepoint, [], $useSTD3ASCIIRules)) {
                $covered = true;
            }
            $this->assertTrue($covered, "Check that code point $codepoint (0x".dechex($codepoint).') is covered');
        }
    }
}

{'fraction_non_alphanumeric': 0.12487611496531219, 'fraction_numerical': 0.004707631318136769, 'mean_word_length': 2.633663366336634, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1975038', 'n_tokens_mistral': 1214, 'n_tokens_neox': 1247, 'n_words': 246}