<commit_before><?php

namespace Igorw\Ilias;

class Reader
{
    public function parse(array $tokens)
    {
        $ast = [];

        for ($i = 0, $length = count($tokens); $i < $length; $i++) {
            $token = $tokens[$i];

            // wrap quoted value
            if ("'" === $token) {
                list($parsedToken, $i) = $this->parseQuotedToken($tokens, $i);
                $ast[] = $parsedToken;
                continue;
            }

            // extract atoms
            while (null !== $token && '(' !== $token && ')' !== $token) {
                $ast[] = $this->normalizeAtom($token);

                $token = isset($tokens[$i+1]) ? $tokens[++$i] : null;
            }

            // handle end condition
            if (null === $token) {
                break;
            }

            // parse list recursively
            if ('(' === $token) {
                list($tokenRange, $i) = $this->getTokenRange($tokens, $i);
                $ast[] = $this->parse($tokenRange);
                continue;
            }
        }

        return $ast;
    }

    private function parseQuotedToken(array $tokens, $i)
    {
        // skip past quote char
        $i++;

        // quoted atom
        if ('(' !== $tokens[$i]) {
            $atom = $this->normalizeAtom($tokens[$i]);
            return [
                new QuotedValue($atom),
                $i,
            ];
        }

        // quoted list
        list($tokenRange, $i) = $this->getTokenRange($tokens, $i);
        $list = $this->parse($tokenRange);

        return [
            new QuotedValue($list),
            $i,
        ];
    }

    private function normalizeAtom($atom)
    {
        if (is_numeric($atom)) {
            return (int) $atom;
        }

        return $atom;
    }

    private function getTokenRange(array $tokens, $i)
    {
        $level = 0;

        $init = $i;
<commit_msg>Rename Reader getTokenRange to extractListTokens<commit_after><?php

namespace Igorw\Ilias;

class Reader
{
    public function parse(array $tokens)
    {
        $ast = [];

        for ($i = 0, $length = count($tokens); $i < $length; $i++) {
            $token = $tokens[$i];

            // wrap quoted value
            if ("'" === $token) {
                list($parsedToken, $i) = $this->parseQuotedToken($tokens, $i);
                $ast[] = $parsedToken;
                continue;
            }

            // extract atoms
            while (null !== $token && '(' !== $token && ')' !== $token) {
                $ast[] = $this->normalizeAtom($token);

                $token = isset($tokens[$i+1]) ? $tokens[++$i] : null;
            }

            // handle end condition
            if (null === $token) {
                break;
            }

            // parse list recursively
            if ('(' === $token) {
                list($tokenRange, $i) = $this->extractListTokens($tokens, $i);
                $ast[] = $this->parse($tokenRange);
                continue;
            }
        }

        return $ast;
    }

    private function parseQuotedToken(array $tokens, $i)
    {
        // skip past quote char
        $i++;

        // quoted atom
        if ('(' !== $tokens[$i]) {
            $atom = $this->normalizeAtom($tokens[$i]);
            return [
                new QuotedValue($atom),
                $i,
            ];
        }

        // quoted list
        list($tokenRange, $i) = $this->extractListTokens($tokens, $i);
        $list = $this->parse($tokenRange);

        return [
            new QuotedValue($list),
            $i,
        ];
    }

    private function normalizeAtom($atom)
    {
        if (is_numeric($atom)) {
            return (int) $atom;
        }

        return $atom;
    }

    private function extractListTokens(array $tokens, $i)
    {
        $level = 0;
        $init = $i;

{'fraction_non_alphanumeric': 0.15863141524105753, 'fraction_numerical': 0.0015552099533437014, 'mean_word_length': 1.341626213592233, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '893943', 'n_tokens_mistral': 1142, 'n_tokens_neox': 1093, 'n_words': 238}