<commit_before><?php
require_once __DIR__ . '/TmpFileTool.php';

class ComplexTypeTool
{
    const REFANG_REGEX_TABLE = array(
        array(
            'from' => '/^(hxxp|hxtp|htxp|meow|h\[tt\]p)/i',
            'to' => 'http',
            'types' => array('link', 'url')
        ),
        array(
            'from' => '/(\[\.\]|\[dot\]|\(dot\))/',
            'to' => '.',
            'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname')
        ),
        array(
            'from' => '/\[hxxp:\/\/\]/',
            'to' => 'http://',
            'types' => array('link', 'url')
        ),
        array(
            'from' => '/[\@]|\[at\]/',
            'to' => '@',
            'types' => array('email', 'email-src', 'email-dst')
        ),
        array(
            'from' => '/\[:\]/',
            'to' => ':',
            'types' => array('url', 'link')
        )
    );

    const HEX_HASH_TYPES = array(
        32 => array('single' => array('md5', 'imphash', 'x509-fingerprint-md5'), 'composite' => array('filename|md5', 'filename|imphash')),
        40 => array('single' => array('sha1', 'pehash', 'x509-fingerprint-sha1', 'cdhash'), 'composite' => array('filename|sha1', 'filename|pehash')),
        56 => array('single' => array('sha224', 'sha512/224'), 'composite' => array('filename|sha224', 'filename|sha512/224')),
        64 => array('single' => array('sha256', 'authentihash', 'sha512/256', 'x509-fingerprint-sha256'), 'composite' => array('filename|sha256', 'filename|authentihash', 'filename|sha512/256')),
        96 => array('single' => array('sha384'), 'composite' => array('filename|sha384')),
        128 => array('single' => array('sha512'), 'composite' => array('filename|sha512'))
    );

    private $__tlds = null;

    public static function refangValue($value, $type)
    {
        foreach (self::REFANG_REGEX_TABLE as $regex) {
            if (in_array($type, $regex['types'], true)) {<commit_msg>new: [freetext] Add support for ja3-fingerprint-md5 import<commit_after><?php
require_once __DIR__ . '/TmpFileTool.php';

class ComplexTypeTool
{
    const REFANG_REGEX_TABLE = array(
        array(
            'from' => '/^(hxxp|hxtp|htxp|meow|h\[tt\]p)/i',
            'to' => 'http',
            'types' => array('link', 'url')
        ),
        array(
            'from' => '/(\[\.\]|\[dot\]|\(dot\))/',
            'to' => '.',
            'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname')
        ),
        array(
            'from' => '/\[hxxp:\/\/\]/',
            'to' => 'http://',
            'types' => array('link', 'url')
        ),
        array(
            'from' => '/[\@]|\[at\]/',
            'to' => '@',
            'types' => array('email', 'email-src', 'email-dst')
        ),
        array(
            'from' => '/\[:\]/',
            'to' => ':',
            'types' => array('url', 'link')
        )
    );

    const HEX_HASH_TYPES = [
        32 => ['single' => ['md5', 'imphash', 'x509-fingerprint-md5', 'ja3-fingerprint-md5'], 'composite' => ['filename|md5', 'filename|imphash']],
        40 => ['single' => ['sha1', 'pehash', 'x509-fingerprint-sha1', 'cdhash'], 'composite' => ['filename|sha1', 'filename|pehash']],
        56 => ['single' => ['sha224', 'sha512/224'], 'composite' => ['filename|sha224', 'filename|sha512/224']],
        64 => ['single' => ['sha256', 'authentihash', 'sha512/256', 'x509-fingerprint-sha256'], 'composite' => ['filename|sha256', 'filename|authentihash', 'filename|sha512/256']],
        96 => ['single' => ['sha384'], 'composite' => ['filename|sha384']],
        128 => ['single' => ['sha512'], 'composite' => ['filename|sha512']],
    ];

    private $__tlds = null;

    public static function refangValue($value, $type)
    {
        foreach (self::REFANG_REGEX_TABLE as $regex) {
            if (in_array($type, $regex['types'], true)) {
{'fraction_non_alphanumeric': 0.24399591211037303, 'fraction_numerical': 0.041389882473173226, 'mean_word_length': 2.36340206185567, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5916126', 'n_tokens_mistral': 1437, 'n_tokens_neox': 1344, 'n_words': 234}