<commit_before><?php
    $source = '';

    while ($a = fread(STDIN, 1024)) {
        $source .= $a;
    }

    $tokens = token_get_all($source);

    $cachedFunctions = array();
    $cachedFunctionComplex = array();
    $tmpFunc = array();

    $nextStringIsFunc = false;
    $isClousure = true;
    $functionParameters = false;
    $snippetCount = 1;

    foreach($tokens as $token) {
        switch($token[0]) {
            case T_FUNCTION:
                $nextStringIsFunc = true;
                break;
            case T_STRING:
                if($nextStringIsFunc) {
                    $isClousure = false;

                    if ($token[1] !== null) {
                        if (!in_array($token[1], $cachedFunctions)) {
                            $cachedFunctions[] = $token[1];
                            $tmpFunc[] =  $token[1];
                        }
                    } else {
                        $nextStringIsFunc = false;
                    }
                }
                break;
            case '(':
                if ($nextStringIsFunc && !$isClousure) {
                    $nextStringIsFunc = false;
                    $isClousure = true;
                    $functionParameters = true;
                }
                break;
            case ')':
                if ($functionParameters) {
                    $functionParameters = false;
                    $cachedFunctionComplex[] = $tmpFunc;
                    $tmpFunc = array();
                    $snippetCount = 1;
                }
                break;
            case T_VARIABLE:
                if ($functionParameters) {
                    $tmpFunc[] = '${' . $snippetCount++ . ':' . $token[1] . '}';
                }
                break;
        }
    }<commit_msg>Fix validations when try to get the custom functions<commit_after><?php
    $source = '';

    while ($a = fread(STDIN, 1024)) {
        $source .= $a;
    }

    $tokens = token_get_all($source);

    $cachedFunctions = array();
    $cachedFunctionComplex = array();
    $tmpFunc = array();

    $insideFunc = false;
    $isClousure = true;
    $haveParameters = false;
    $snippetCount = 1;

    foreach($tokens as $token) {
        switch($token[0]) {
            case T_FUNCTION:
                $insideFunc = true;
                break;
            case T_STRING:
                if($insideFunc) {
                    $isClousure = false;

                    if ($token[1] !== null) {
                        if (!in_array($token[1], $cachedFunctions)) {
                            $cachedFunctions[] = $token[1];
                            $tmpFunc[] =  $token[1];
                        }
                    } else {
                        $insideFunc = false;
                    }
                }
                break;
            case '(':
                if ($insideFunc && !$isClousure) {
                    $haveParameters = true;
                }
                break;
            case ')':
                if ($insideFunc && $haveParameters) {
                    $cachedFunctionComplex[] = $tmpFunc;
                    $tmpFunc = array();
                    $snippetCount = 1;
                }

                $haveParameters = false;
                $isClousure = true;
                $insideFunc = false;
                break;
            case T_VARIABLE:
                if ($haveParameters) {
                    $tmpFunc[] = '${' . $snippetCount++ . ':' . $token[1] . '}';
                }
                break;
        }
    }
{'fraction_non_alphanumeric': 0.11905432029271039, 'fraction_numerical': 0.006754855052068675, 'mean_word_length': 1.0239179954441913, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4913994', 'n_tokens_mistral': 976, 'n_tokens_neox': 884, 'n_words': 175}