<commit_before><?php
require_once __DIR__ . '/../Lib/Tools/AttributeValidationTool.php';

use PHPUnit\Framework\TestCase;

if (!function_exists('__')) {
    // Fake translation function
    function __($singular, $args = null)
    {
        $arguments = func_get_args();
        return vsprintf($singular, array_slice($arguments, 1));
    }
}

class AttributeValidationToolTest extends TestCase
{
    public function testValidateHash(): void
    {
        $this->shouldBeValid('filename|md5', [
            'cmd.exe|0cc175b9c0f1b6a831c399e269772661',
        ]);
        $this->shouldBeInvalid('filename|md5', [
            'cmd.exe|86f7e437faa5a7fce15d1ddcb9eaeaea377667b8',
        ]);
    }

    public function testValidateIp(): void
    {
        foreach (['ip-src', 'ip-dst'] as $type) {
            $this->shouldBeValid($type, [
                '127.0.0.1',
                '127.0.0.1/32',
                '::1',
                '::1/128',
            ]);
            $this->shouldBeInvalid($type, [
                '127',
                '127.0.0.',
                '127.0.0.1/',
                '127.0.0.1/32/1',
                '127.0.0.1/128',
                '::1/257',
                '::1/257',
                '::1/128/1',
            ]);
        }
    }

    public function testValidatePort(): void
    {
        $this->assertTrue(AttributeValidationTool::validate('port', '1'));
        $this->assertTrue(AttributeValidationTool::validate('port', 1));
        $this->assertTrue(AttributeValidationTool::validate('port', 80));
        $this->assertNotTrue(AttributeValidationTool::validate('port', -80));
        $this->assertNotTrue(AttributeValidationTool::validate('port', '-80'));
    }

    public function testValidateSshFingerprint(): void
    {
        $this->shouldBeValid('ssh-fingerprint', [
            '7b:e5:6f:a7:f4:f9:81:62:5c:e3:1f:bf:8b:57:6c:5a',
            'MD5:7b:e5:6f:a7:f4:f9:81:62:5c:e3:1f:bf:8b:57:6c:5a',
            'SHA256:mVPwvezndPv/ARoIadVY98vAC0g+P/5633yTC4d/wXE',
        ]);
    }

    public function testValidateSsdeep(): void
    {
        $this->shouldBeValid('ssdeep', [
            '96:s4Ud1Lj96tHHlZDrwciQmA+4uy1I0G4HYuL8N3TzS8QsO/wqWXLcMSx:sF1LjEtHHlZDrJzrhuyZvHYm8tKp/RWO',
            '384:EWo4X1WaPW9ZWhWzLo+lWpct/fWbkWsWIwW0/S7dZhgG8:EWo4X1WmW9ZWhWH/WpchfWgWsWTWtf8',
            '6144:3wSQSlrBHFjOvwYAU/Fsgi/2WDg5+YaNk5xcHrYw+Zg+XrZsGEREYRGAFU25ttR/:ctM7E0L4q',
        ]);
        $this->shouldBeValid('filename|ssdeep', [
            'ahoj.txt|96:s4Ud1Lj96tHHlZDrwciQmA+4uy1I0G4HYuL8N3TzS8QsO/wqWXLcMSx:sF1LjEtHHlZDrJzrhuyZvHYm8tKp/RWO',
        ]);
    }

    public function testValidateDomainIp(): void
    {
        $this->shouldBeValid('domain|ip', [
            'example.com|127.0.0.1',
            'example.com|::1',
        ]);
        $this->shouldBeInvalid('domain|ip', [
            'example.com|127',
            'example.com|1',
        ]);
    }

    public function testValidateFilename(): void
    {
        $this->shouldBeValid('filename', [
            'cmd.exe',
            'cmd.com',
        ]);
        $this->shouldBeInvalid('filename', [
            "cmd.exe\ncmd.com",
        ]);
        $this->shouldBeValid('filename|md5', [
            'cmd.exe|0cc175b9c0f1b6a831c399e269772661',
            'cmd.com|0cc175b9c0f1b6a831c399e269772661',
        ]);
        $this->shouldBeInvalid('filename|md5', [
            "cmd.exe\ncmd.com|0cc175b9c0f1b6a831c399e269772661",
        ]);
    }

    public function testCompressIpv6(): void
    {
        $this->assertEquals('fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b', AttributeValidationTool::modifyBeforeValidation('ip-src', 'fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b'));
        $this->assertEquals('example.com|fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'example.com|fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b'));
        $this->assertEquals('fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', 'fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b|80'));
        $this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1'));
    }

    public function testFilenameHashLowercase()
    {
        $this->assertEquals('CMD.EXE|0cc175b9c0f1b6a831c399e269772661', AttributeValidationTool::modifyBeforeValidation('filename|md5', 'CMD.EXE|0CC175B9C0F1B6A831C399E269772661'));
    }

    private function shouldBeValid($type, array $values)
    {
        foreach ($values as $value) {
            $this->assertTrue(AttributeValidationTool::validate($type, $value));
        }
    }

    private function shouldBeInvalid($type, array $values)
    {
        foreach ($values as $value) {
            $this->assertNotTrue(AttributeValidationTool::validate($type, $value));
        }
    }
}
<commit_msg>chg: [test] testDomainModify<commit_after><?php
require_once __DIR__ . '/../Lib/Tools/AttributeValidationTool.php';

use PHPUnit\Framework\TestCase;

if (!function_exists('__')) {
    // Fake translation function
    function __($singular, $args = null)
    {
        $arguments = func_get_args();
        return vsprintf($singular, array_slice($arguments, 1));
    }
}

class AttributeValidationToolTest extends TestCase
{
    public function testValidateHash(): void
    {
        $this->shouldBeValid('filename|md5', [
            'cmd.exe|0cc175b9c0f1b6a831c399e269772661',
        ]);
        $this->shouldBeInvalid('filename|md5', [
            'cmd.exe|86f7e437faa5a7fce15d1ddcb9eaeaea377667b8',
        ]);
    }

    public function testValidateIp(): void
    {
        foreach (['ip-src', 'ip-dst'] as $type) {
            $this->shouldBeValid($type, [
                '127.0.0.1',
                '127.0.0.1/32',
                '::1',
                '::1/128',
            ]);
            $this->shouldBeInvalid($type, [
                '127',
                '127.0.0.',
                '127.0.0.1/',
                '127.0.0.1/32/1',
                '127.0.0.1/128',
                '::1/257',
                '::1/257',
                '::1/128/1',
            ]);
        }
    }

    public function testValidatePort(): void
    {
        $this->assertTrue(AttributeValidationTool::validate('port', '1'));
        $this->assertTrue(AttributeValidationTool::validate('port', 1));
        $this->assertTrue(AttributeValidationTool::validate('port', 80));
        $this->assertNotTrue(AttributeValidationTool::validate('port', -80));
        $this->assertNotTrue(AttributeValidationTool::validate('port', '-80'));
    }

    public function testValidateSshFingerprint(): void
    {
        $this->shouldBeValid('ssh-fingerprint', [
            '7b:e5:6f:a7:f4:f9:81:62:5c:e3:1f:bf:8b:57:6c:5a',
            'MD5:7b:e5:6f:a7:f4:f9:81:62:5c:e3:1f:bf:8b:57:6c:5a',
            'SHA256:mVPwvezndPv/ARoIadVY98vAC0g+P/5633yTC4d/wXE',
        ]);
    }

    public function testValidateSsdeep(): void
    {
        $this->shouldBeValid('ssdeep', [
            '96:s4Ud1Lj96tHHlZDrwciQmA+4uy1I0G4HYuL8N3TzS8QsO/wqWXLcMSx:sF1LjEtHHlZDrJzrhuyZvHYm8tKp/RWO',
            '384:EWo4X1WaPW9ZWhWzLo+lWpct/fWbkWsWIwW0/S7dZhgG8:EWo4X1WmW9ZWhWH/WpchfWgWsWTWtf8',
            '6144:3wSQSlrBHFjOvwYAU/Fsgi/2WDg5+YaNk5xcHrYw+Zg+XrZsGEREYRGAFU25ttR/:ctM7E0L4q',
        ]);
        $this->shouldBeValid('filename|ssdeep', [
            'ahoj.txt|96:s4Ud1Lj96tHHlZDrwciQmA+4uy1I0G4HYuL8N3TzS8QsO/wqWXLcMSx:sF1LjEtHHlZDrJzrhuyZvHYm8tKp/RWO',
        ]);
    }

    public function testValidateDomainIp(): void
    {
        $this->shouldBeValid('domain|ip', [
            'example.com|127.0.0.1',
            'example.com|::1',
        ]);
        $this->shouldBeInvalid('domain|ip', [
            'example.com|127',
            'example.com|1',
        ]);
    }

    public function testValidateFilename(): void
    {
        $this->shouldBeValid('filename', [
            'cmd.exe',
            'cmd.com',
        ]);
        $this->shouldBeInvalid('filename', [
            "cmd.exe\ncmd.com",
        ]);
        $this->shouldBeValid('filename|md5', [
            'cmd.exe|0cc175b9c0f1b6a831c399e269772661',
            'cmd.com|0cc175b9c0f1b6a831c399e269772661',
        ]);
        $this->shouldBeInvalid('filename|md5', [
            "cmd.exe\ncmd.com|0cc175b9c0f1b6a831c399e269772661",
        ]);
    }

    public function testCompressIpv6(): void
    {
        $this->assertEquals('1234:fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b', AttributeValidationTool::modifyBeforeValidation('ip-src', 'fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b'));
        $this->assertEquals('example.com|fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'example.com|1234:0fd2:5621:0001:0089:0000:0000:4500'));
        $this->assertEquals('fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', 'fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b|80'));
        $this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1'));
    }

    public function testFilenameHashLowercase()
    {
        $this->assertEquals('CMD.EXE|0cc175b9c0f1b6a831c399e269772661', AttributeValidationTool::modifyBeforeValidation('filename|md5', 'CMD.EXE|0CC175B9C0F1B6A831C399E269772661'));
    }

    public function testDomainModify()
    {
        $this->assertEquals('example.com', AttributeValidationTool::modifyBeforeValidation('domain', 'example.com'));
        $this->assertEquals('example.com', AttributeValidationTool::modifyBeforeValidation('domain', 'EXAMPLE.COM'));
        $this->assertEquals('example.com|127.0.0.1', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'example.com|127.0.0.1'));
        $this->assertEquals('example.com|127.0.0.1', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'EXAMPLE.COM|127.0.0.1'));
        $this->assertEquals('xn--hkyrky-ptac70bc.cz', AttributeValidationTool::modifyBeforeValidation('domain', 'háčkyčárky.cz'));
        $this->assertEquals('xn--hkyrky-ptac70bc.cz', AttributeValidationTool::modifyBeforeValidation('domain', 'HÁČKYČÁRKY.CZ'));
        $this->assertEquals('xn--hkyrky-ptac70bc.cz|127.0.0.1', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'háčkyčárky.cz|127.0.0.1'));
        $this->assertEquals('xn--hkyrky-ptac70bc.cz|127.0.0.1', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'HÁČKYČÁRKY.CZ|127.0.0.1'));
    }

    private function shouldBeValid($type, array $values)
    {
        foreach ($values as $value) {
            $this->assertTrue(AttributeValidationTool::validate($type, $value));
        }
    }

    private function shouldBeInvalid($type, array $values)
    {
        foreach ($values as $value) {
            $this->assertNotTrue(AttributeValidationTool::validate($type, $value));
        }
    }
}

{'fraction_non_alphanumeric': 0.15947880345017434, 'fraction_numerical': 0.09478803450174345, 'mean_word_length': 2.9106566200215287, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 24, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4878632', 'n_tokens_mistral': 4543, 'n_tokens_neox': 4096, 'n_words': 341}