<commit_before><?php
/**
 * For licensing information, please see the LICENSE file accompanied with this file.
 *
 * @author <NAME> <<EMAIL>>
 * @copyright 2012 <NAME> <http://melp.nl>
 */

namespace Zicht\Tool\Script\Parser;

use \Zicht\Tool\Script\Token;
use \Zicht\Tool\Script\Node;
use \Zicht\Tool\Script\Node\Expr\Op;
use \Zicht\Tool\Script\AbstractParser;
use \Zicht\Tool\Script\TokenStream;
/**
 * Expression parser
 */
class Expression extends AbstractParser
{
    /**
     * Unary prefix operators
     *
     * @var array
     */
    public static $PREFIX_UNARY = array('!', '-', '~');

    /**
     * Binary infix operators
     *
     * @var array
     */
    public static $INFIX_BINARY = array(
        '==', '!=', '<=', '>=', '<', '>', '&&', '||',
        'cat' // 'cat' is a hack for BC supporting the dot operator for concatenation
    );


    /**
     * Does a recursive descent parsing of the token stream and returns the resulting node.
     *
     * @param \Zicht\Tool\Script\TokenStream $stream
     * @return \Zicht\Tool\Script\Node\Node
     */
    public function parse(TokenStream $stream)
    {
        if ($stream->match(Token::LEGACY_ENV)) {
            $stream->next();

            if ($stream->match(Token::OPERATOR, '.')) {
                $ret = new Node\Expr\Subscript(new Node\Expr\Variable('envs'));
                $ret->append(new Node\Expr\Variable('target_env'));
            } else {
                $ret = new Node\Expr\Variable('target_env');
            }
        } elseif ($stream->match(Token::OPERATOR, array('!', '-'))) {
            $value = $stream->current()->value;
            $stream->next();
            $ret = new Op\Unary($value, $this->parse($stream));
        } elseif ($stream->match(Token::IDENTIFIER)) {
            $name = $stream->current()->value;
            $stream->next();

            $ret = new Node\Expr\Variable($name);
        } elseif ($stream->match(Token::STRING)) {
            $ret = new Node\Expr\Str($stream->current()->value);<commit_msg>added some explanation to the LEGACY_ENV token<commit_after><?php
/**
 * For licensing information, please see the LICENSE file accompanied with this file.
 *
 * @author <NAME> <<EMAIL>>
 * @copyright 2012 <NAME> <http://melp.nl>
 */

namespace Zicht\Tool\Script\Parser;

use \Zicht\Tool\Script\Token;
use \Zicht\Tool\Script\Node;
use \Zicht\Tool\Script\Node\Expr\Op;
use \Zicht\Tool\Script\AbstractParser;
use \Zicht\Tool\Script\TokenStream;
/**
 * Expression parser
 */
class Expression extends AbstractParser
{
    /**
     * Unary prefix operators
     *
     * @var array
     */
    public static $PREFIX_UNARY = array('!', '-', '~');

    /**
     * Binary infix operators
     *
     * @var array
     */
    public static $INFIX_BINARY = array(
        '==', '!=', '<=', '>=', '<', '>', '&&', '||',
        'cat' // 'cat' is a hack for BC supporting the dot operator for concatenation
    );


    /**
     * Does a recursive descent parsing of the token stream and returns the resulting node.
     *
     * @param \Zicht\Tool\Script\TokenStream $stream
     * @return \Zicht\Tool\Script\Node\Node
     */
    public function parse(TokenStream $stream)
    {
        // BC feature: if a legacy env token is matched, replace it with envs[target_env]
        if ($stream->match(Token::LEGACY_ENV)) {
            $stream->next();

            if ($stream->match(Token::OPERATOR, '.')) {
                $ret = new Node\Expr\Subscript(new Node\Expr\Variable('envs'));
                $ret->append(new Node\Expr\Variable('target_env'));
            } else {
                $ret = new Node\Expr\Variable('target_env');
            }
        } elseif ($stream->match(Token::OPERATOR, array('!', '-'))) {
            $value = $stream->current()->value;
            $stream->next();
            $ret = new Op\Unary($value, $this->parse($stream));
        } elseif ($stream->match(Token::IDENTIFIER)) {
            $name = $stream->current()->value;
            $stream->next();

            $ret = new Node\Expr\Variable($name);
        } elseif ($stream->match(Token::STRING)) {
            $ret = new Node\Expr\Str($stream->current()->value);
{'fraction_non_alphanumeric': 0.17278617710583152, 'fraction_numerical': 0.001919846412287017, 'mean_word_length': 2.3211155378486055, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4430464', 'n_tokens_mistral': 1270, 'n_tokens_neox': 1242, 'n_words': 278}