<commit_before><?php
namespace Phpforce\Query;

use Phpforce\Query\AST as AST;
use Doctrine\Common\Cache\Cache;
use Symfony\Component\Yaml\Exception\ParseException;

class Parser
{
    const
        SCOPE_SELECT = 'SCOPE_SELECT',
        SCOPE_FROM = 'SCOPE_FROM',
        SCOPE_WHERE = 'SCOPE_WHERE',
        SCOPE_WITH = 'SCOPE_WITH',
        SCOPE_GROUPBY = 'SCOPE_GROUPBY',
        SCOPE_HAVING = 'SCOPE_HAVING',
        SCOPE_ORDERBY = 'ORDERBY'
    ;

    /**
     * @var Tokenizer
     */
    private $tokenizer;

    /**
     * @var string
     */
    private $scope;

    /**
     * @var FunctionDefinition[]
     */
    private static $availableFunctions;

    /**
     * @var bool
     */
    private $isSubquery = false;

    /**
     * @var Cache
     */
    private $cache;

    /**
     * @param \Phpforce\Query\TokenizerInterface $tokenizer
     * @param Cache $cache
     */
    public function __construct(TokenizerInterface $tokenizer, Cache $cache)
    {
        $this->cache = $cache;

        $this->tokenizer = $tokenizer;

        static::$availableFunctions = array
        (
            'TOLABEL' => new FunctionDefinition(FunctionDefinition::TYPE_STRING, array(static::SCOPE_SELECT, static::SCOPE_WHERE), 1),
            'CONVERTCURRENCY' => new FunctionDefinition(FunctionDefinition::TYPE_STRING, array(static::SCOPE_SELECT), 1),

            'CONVERTTIMEZONE' => new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),

            'CUBE' => new FunctionDefinition(FunctionDefinition::TYPE_GROUPING, array(static::SCOPE_GROUPBY), 1, -1),
            'ROLLUP' => new FunctionDefinition(FunctionDefinition::TYPE_GROUPING, array(static::SCOPE_GROUPBY), 1, -1),

            'GROUPING' => new FunctionDefinition(FunctionDefinition::TYPE_AGGREGATE, array(static::SCOPE_SELECT, static::SCOPE_ORDERBY, static::SCOPE_HAVING), 1),
            'COUNT' => new FunctionDefinition(FunctionDefinition::TYPE_AGGREGATE, array(static::SCOPE_SELECT, static::SCOPE_ORDERBY, static::SCOPE_HAVING), 0, 1),
            'AVG' => new FunctionDefinition(FunctionDefinition::TYPE_AGGREGATE, array(static::SCOPE_SELECT, static::SCOPE_ORDERBY, static::SCOPE_HAVING), 1),
            'COUNT_DISTINCT'=> new FunctionDefinition(FunctionDefinition::TYPE_AGGREGATE, array(static::SCOPE_SELECT, static::SCOPE_ORDERBY, static::SCOPE_HAVING), 1),
            'MIN'=> new FunctionDefinition(FunctionDefinition::TYPE_AGGREGATE, array(static::SCOPE_SELECT, static::SCOPE_ORDERBY, static::SCOPE_HAVING), 1),
            'MAX'=> new FunctionDefinition(FunctionDefinition::TYPE_AGGREGATE, array(static::SCOPE_SELECT, static::SCOPE_ORDERBY, static::SCOPE_HAVING), 1),
            'SUM'=> new FunctionDefinition(FunctionDefinition::TYPE_AGGREGATE, array(static::SCOPE_SELECT, static::SCOPE_ORDERBY, static::SCOPE_HAVING), 1),

            'CALENDAR_MONTH'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'CALENDAR_QUARTER'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'CALENDAR_YEAR'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'DAY_IN_MONTH'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'DAY_IN_WEEK'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'DAY_IN_YEAR'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'DAY_ONLY'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'FISCAL_MONTH'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'FISCAL_QUARTER'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'FISCAL_YEAR'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'HOUR_IN_DAY'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'WEEK_IN_MONTH'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'WEEK_IN_YEAR'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1)
        );
    }

    /**
     * @param $soql
     * @return AST\Node|AST\Query|null
     */
    public function parse($soql)
    {
        $id = hash('sha1', $soql);

        if($this->cache->contains($id))
        {
            $this->query = $this->cache->fetch($id);
        }
        else
        {
            $this->tokenizer->tokenize($soql);

            $this->tokenizer->proceedSkip();

            $this->query = $this->parseQuery();

            $this->cache->save($id, $this->query);
        }
        return $this->query;
    }

    /**
     * @return AST\Query
     */
    public function parseQuery()
    {
        $this->tokenizer->checkKeyword('SELECT');

        $this->scope = static::SCOPE_SELECT;

        $query = new AST\Query();

        $query->select = $this->parseSelectFieldList();

        $this->tokenizer->checkKeyword('FROM');

        $this->scope = static::SCOPE_FROM;

        $this->tokenizer->expect(TokenDefinition::T_EXPRESSION);

        $from = $this->tokenizer->getValue();

        $this->tokenizer->proceedSkip();

        $alias = $this->parseAlias();

        $query->from = new AST\From($from, $alias);

        $this->tokenizer->check(array(
            TokenDefinition::T_KEYWORD,
            TokenDefinition::T_EOQ,
            TokenDefinition::T_RIGHT_PAREN
        ));

        if($this->tokenizer->isKeyword('WHERE'))
        {
            $this->scope = static::SCOPE_WHERE;

            $this->tokenizer->proceedSkip();

            $query->where = new AST\Where($this->parseLogicalGroup());
        }

        $this->tokenizer->check(array(
            TokenDefinition::T_KEYWORD,
            TokenDefinition::T_EOQ,
            TokenDefinition::T_RIGHT_PAREN
        ));

        if($this->tokenizer->isKeyword('WITH'))
        {
            $this->scope = static::SCOPE_WITH;

            $this->tokenizer->proceedSkip();

            if($this->tokenizer->isKeyword('DATA'))
            {
                $this->tokenizer->expectKeyword('CATEGORY');

                $query->with = new AST\With($this->parseWithDataCategory(), AST\With::DATA_CATEGORY);


            }
            else
            {
                $query->with = new AST\With($this->parseLogicalGroup());
            }
        }

        $this->tokenizer->check(array(
            TokenDefinition::T_KEYWORD,
            TokenDefinition::T_EOQ,<commit_msg>Removed whitespace<commit_after><?php
namespace Phpforce\Query;

use Phpforce\Query\AST as AST;
use Doctrine\Common\Cache\Cache;
use Symfony\Component\Yaml\Exception\ParseException;

class Parser
{
    const
        SCOPE_SELECT = 'SCOPE_SELECT',
        SCOPE_FROM = 'SCOPE_FROM',
        SCOPE_WHERE = 'SCOPE_WHERE',
        SCOPE_WITH = 'SCOPE_WITH',
        SCOPE_GROUPBY = 'SCOPE_GROUPBY',
        SCOPE_HAVING = 'SCOPE_HAVING',
        SCOPE_ORDERBY = 'ORDERBY'
    ;

    /**
     * @var Tokenizer
     */
    private $tokenizer;

    /**
     * @var string
     */
    private $scope;

    /**
     * @var FunctionDefinition[]
     */
    private static $availableFunctions;

    /**
     * @var bool
     */
    private $isSubquery = false;

    /**
     * @var Cache
     */
    private $cache;

    /**
     * @param \Phpforce\Query\TokenizerInterface $tokenizer
     * @param Cache $cache
     */
    public function __construct(TokenizerInterface $tokenizer, Cache $cache)
    {
        $this->cache = $cache;

        $this->tokenizer = $tokenizer;

        static::$availableFunctions = array
        (
            'TOLABEL' => new FunctionDefinition(FunctionDefinition::TYPE_STRING, array(static::SCOPE_SELECT, static::SCOPE_WHERE), 1),
            'CONVERTCURRENCY' => new FunctionDefinition(FunctionDefinition::TYPE_STRING, array(static::SCOPE_SELECT), 1),

            'CONVERTTIMEZONE' => new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),

            'CUBE' => new FunctionDefinition(FunctionDefinition::TYPE_GROUPING, array(static::SCOPE_GROUPBY), 1, -1),
            'ROLLUP' => new FunctionDefinition(FunctionDefinition::TYPE_GROUPING, array(static::SCOPE_GROUPBY), 1, -1),

            'GROUPING' => new FunctionDefinition(FunctionDefinition::TYPE_AGGREGATE, array(static::SCOPE_SELECT, static::SCOPE_ORDERBY, static::SCOPE_HAVING), 1),
            'COUNT' => new FunctionDefinition(FunctionDefinition::TYPE_AGGREGATE, array(static::SCOPE_SELECT, static::SCOPE_ORDERBY, static::SCOPE_HAVING), 0, 1),
            'AVG' => new FunctionDefinition(FunctionDefinition::TYPE_AGGREGATE, array(static::SCOPE_SELECT, static::SCOPE_ORDERBY, static::SCOPE_HAVING), 1),
            'COUNT_DISTINCT'=> new FunctionDefinition(FunctionDefinition::TYPE_AGGREGATE, array(static::SCOPE_SELECT, static::SCOPE_ORDERBY, static::SCOPE_HAVING), 1),
            'MIN'=> new FunctionDefinition(FunctionDefinition::TYPE_AGGREGATE, array(static::SCOPE_SELECT, static::SCOPE_ORDERBY, static::SCOPE_HAVING), 1),
            'MAX'=> new FunctionDefinition(FunctionDefinition::TYPE_AGGREGATE, array(static::SCOPE_SELECT, static::SCOPE_ORDERBY, static::SCOPE_HAVING), 1),
            'SUM'=> new FunctionDefinition(FunctionDefinition::TYPE_AGGREGATE, array(static::SCOPE_SELECT, static::SCOPE_ORDERBY, static::SCOPE_HAVING), 1),

            'CALENDAR_MONTH'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'CALENDAR_QUARTER'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'CALENDAR_YEAR'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'DAY_IN_MONTH'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'DAY_IN_WEEK'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'DAY_IN_YEAR'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'DAY_ONLY'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'FISCAL_MONTH'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'FISCAL_QUARTER'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'FISCAL_YEAR'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'HOUR_IN_DAY'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'WEEK_IN_MONTH'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1),
            'WEEK_IN_YEAR'=> new FunctionDefinition(FunctionDefinition::TYPE_DATE, array(static::SCOPE_SELECT, static::SCOPE_GROUPBY, static::SCOPE_ORDERBY, static::SCOPE_WHERE, static::SCOPE_HAVING), 1)
        );
    }

    /**
     * @param $soql
     * @return AST\Node|AST\Query|null
     */
    public function parse($soql)
    {
        $id = hash('sha1', $soql);

        if($this->cache->contains($id))
        {
            $this->query = $this->cache->fetch($id);
        }
        else
        {
            $this->tokenizer->tokenize($soql);

            $this->tokenizer->proceedSkip();

            $this->query = $this->parseQuery();

            $this->cache->save($id, $this->query);
        }
        return $this->query;
    }

    /**
     * @return AST\Query
     */
    public function parseQuery()
    {
        $this->tokenizer->checkKeyword('SELECT');

        $this->scope = static::SCOPE_SELECT;

        $query = new AST\Query();

        $query->select = $this->parseSelectFieldList();

        $this->tokenizer->checkKeyword('FROM');

        $this->scope = static::SCOPE_FROM;

        $this->tokenizer->expect(TokenDefinition::T_EXPRESSION);

        $from = $this->tokenizer->getValue();

        $this->tokenizer->proceedSkip();

        $alias = $this->parseAlias();

        $query->from = new AST\From($from, $alias);

        $this->tokenizer->check(array(
            TokenDefinition::T_KEYWORD,
            TokenDefinition::T_EOQ,
            TokenDefinition::T_RIGHT_PAREN
        ));

        if($this->tokenizer->isKeyword('WHERE'))
        {
            $this->scope = static::SCOPE_WHERE;

            $this->tokenizer->proceedSkip();

            $query->where = new AST\Where($this->parseLogicalGroup());
        }

        $this->tokenizer->check(array(
            TokenDefinition::T_KEYWORD,
            TokenDefinition::T_EOQ,
            TokenDefinition::T_RIGHT_PAREN
        ));

        if($this->tokenizer->isKeyword('WITH'))
        {
            $this->scope = static::SCOPE_WITH;

            $this->tokenizer->proceedSkip();

            if($this->tokenizer->isKeyword('DATA'))
            {
                $this->tokenizer->expectKeyword('CATEGORY');

                $query->with = new AST\With($this->parseWithDataCategory(), AST\With::DATA_CATEGORY);
            }
            else
            {
                $query->with = new AST\With($this->parseLogicalGroup());
            }
        }

        $this->tokenizer->check(array(
            TokenDefinition::T_KEYWORD,
            TokenDefinition::T_EOQ,
{'fraction_non_alphanumeric': 0.13878624663368197, 'fraction_numerical': 0.0036324920147804847, 'mean_word_length': 3.6310904872389793, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 171, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3719216', 'n_tokens_mistral': 5242, 'n_tokens_neox': 4829, 'n_words': 694}