<commit_before><?php

/*
 * This file is part of the 'octris/core' package.
 *
 * (c) <NAME> <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Octris\Core\Parser;

/**
 * Class for defining a parser grammar.
 *
 * @copyright   copyright (c) 2014 by <NAME>
 * @author      <NAME> <<EMAIL>>
 */
class Grammar
{
    /**
     * Unknown token.
     *
     * @type    int
     */
    const T_UNKNOWN = 0;

    /**
     * ID of initial rule.
     *
     * @type    int|string|null
     */
    protected $initial = null;

    /**
     * Grammar rules.
     *
     * @type    array
     */
    protected $rules = [];

    /**
     * Events for tokens.
     *
     * @type    array
     */
    protected $events = [];

    /**
     * Registered tokens.
     *
     * @type    array
     */
    protected $tokens = array();

    /**
     * Constructor.
     */
    public function __construct()
    {
    }

    /**
     * Add a rule to the grammar.
     *
     * @param   int|string          $id                 Token identifier to apply the rule for.
     * @param   array               $rule               Grammar rule.
     * @param   bool                $initial            Whether to set the rule as initial.
     */
    public function addRule($id, array $rule, $initial = false)
    {
        if (isset($this->rules[$id])) {
            throw new \Exception('Rule is already defined!');
        }

        // { validate
        $iterator = new \RecursiveIteratorIterator(
            new \RecursiveArrayIterator($rule),
            true
        );

        foreach ($iterator as $k => $v) {
            if (!is_int($k)) {
                if (!is_array($v)) {
                    throw new \Exception(sprintf("No array specified for rule operator '%s'", $k));
                }

                switch ($k) {
                    case '$option':
                        if (($cnt = count($v)) != 1) {
                            throw new \Exception(sprintf("Rule operator '\$option' takes exactly one item, '%d' given", $cnt));
                        }
                        break;
                    case '$alternation':
                    case '$concatenation':
                    case '$repeat':
                        break;
                    default:
                        throw new \Exception(sprintf("Invalid rule operator '%s'", $k));
                }
            }
        }
        // }

        $this->rules[$id] = $rule;

        if ($initial) {
            $this->initial = $id;
        }
    }

    /**
     * Add an event for a token.
     *
     * @param   int                 $id                 Token identifier.
     * @param   callable            $cb                 Callback to call if the token occurs.
     */
    public function addEvent($id, callable $cb)
    {
        if (!isset($this->events[$id])) {
            $this->events[$id] = [];
        }

        $this->events[$id][] = $cb;
    }

    /**
     * Return list of defined tokens.
     *
     * @return  array                                   Defined tokens.
     */
    public function getTokens()
    {
        return $this->tokens;
    }

    /**
     * Return names of tokens. Will only work, if tokens are defined using class 'constants'.
     *
     * @return  array                                   Names of defined tokens.
     */
    public function getTokenNames()
    {
        return array_flip((new \ReflectionClass($this))->getConstants());
    }

    /**
     * Add a token to the registry.
     *
     * @param   string          $name                   Name of token.
     * @param   string          $regexp                 Regular expression for parser to match token.
     */
    public function addToken($name, $regexp)
    {
        $this->tokens[$name] = $regexp;
    }

    /**
     * Return the EBNF for the defined grammar.
     *
     * @return  string                                  The EBNF.
     */
    public function getEBNF()
    {
        $glue = array(
            '$concatenation' => array('', ' , ', ''),
            '$alternation'   => array('( ', ' | ', ' )'),
            '$repeat'        => array('{ ', '', ' }'),
            '$option'        => array('[ ', '', ' ]')
        );

        $render = function ($rule) use ($glue, &$render) {
            if (is_array($rule)) {
                $type = key($rule);

                foreach ($rule[$type] as &$_rule) {
                    $_rule = $render($_rule);
                }

                $return = $glue[$type][0] .
                          implode($glue[$type][1], $rule[$type]) .
                          $glue[$type][2];
            } else {
                $return = $rule;
            }

            return $return;
        };

        $return = '';

        foreach ($this->rules as $name => $rule) {
            $return .= $name . ' = ' . $render($rule) . " ;\n";
        }

        return $return;
    }

    /**
     * Analyze / validate token stream. If the token stream is invalid, the second, optional, parameter
     * will contain the expected token.
     *
     * @param   array               $tokens             Token stream to analyze.
     * @param   array               &$error             If an error occured the variable get's filled with the current token information and expected token(s).
     * @return  bool                                    Returns true if token stream is valid compared to the defined grammar.
     */
    public function analyze($tokens, &$error = null)
    {
        $expected = [];
        $pos      = 0;
        $error    = false;

        $v = function ($rule) use ($tokens, &$pos, &$v, &$expected, &$error) {
            $valid = false;

            if (is_scalar($rule) && isset($this->rules[$rule])) {
                // import rule
                $rule = $this->rules[$rule];
            }

            if (is_array($rule)) {
                $type = key($rule);

                switch ($type) {
                    case '$concatenation':
                        $state = $pos;

                        foreach ($rule[$type] as $_rule) {
                            if (!($valid = $v($_rule))) {
                                if ($error) {
                                    return false;
                                } elseif (($pos - $state) > 0) {
                                    $error = (isset($tokens[$pos])
                                              ? $tokens[$pos]
                                              : array_merge(
                                                  $tokens[$pos - 1],
                                                  array(
                                                      'token' => self::T_UNKNOWN,
                                                      'value' => self::T_UNKNOWN
                                                  )
                                              ));

                                    $error['expected'] = array_unique($expected);
                                    return false;
                                }
                                break;
                            }
                        }

                        if (!$valid) {
                            // rule did not match, restore position in token stream
                            $pos   = $state;
                            $valid = false;
                        }
                        break;
                    case '$alternation':
                        $state = $pos;

                        foreach ($rule[$type] as $_rule) {
                            if (($valid = $v($_rule)) || $error) {
                                // if ($error) return false;
                                break;
                            }
                        }

                        if (!$valid) {
                            // rule did not match, restore position in token stream
                            $pos   = $state;
                            $valid = false;
                        }
                        break;
                    case '$option':
                        $state = $pos;

                        foreach ($rule[$type] as $_rule) {
                            if (($valid = $v($_rule)) || $error) {
                                if ($error) {
                                    return false;
                                }
                                break;
                            }
                        }

                        if (!$valid) {
                            // rule did not match, restore position in token stream
                            $pos   = $state;
                            $valid = true;
                        }
                        break;
                    case '$repeat':
                        do {
                            $state = $pos;

                            foreach ($rule[$type] as $_rule) {
                                if (($valid = $v($_rule)) || $error) {
                                    if ($error) {
                                        return false;
                                    }
                                    break;
                                }
                            }
                        } while ($valid);

                        if (!$valid) {
                            // rule did not match, restore position in token stream
                            $pos   = $state;
                            $valid = true;
                        }
                        break;
                }
            } elseif (($valid = isset($tokens[$pos]))) {
                $token = $tokens[$pos];

                if (($valid = ($token['token'] == $rule))) {
                    ++$pos;
                    $expected = [];
                } else {
                    $expected[] = $rule;
                }
            } else {
                $expected[] = $rule;
            }

            return (!$error ? $valid : false);
        };

        if (!is_null($this->initial)) {
            $valid = $v($this->rules[$this->initial]);
        } else {
            // no initial rule, build one
            $valid = $v(['$alternation' => array_keys($this->rules)]);
        }

        $valid = (!$error && $valid);

        if ($valid) {
            foreach ($tokens as $token) {
                if (isset($this->events[$token['token']])) {
                    foreach ($this->events[$token['token']] as $event) {
                        $event($token);
                    }
                }
            }
        } elseif (!$error) {
            $error = $tokens[$pos];
            $error['expected'] = $expected;
        }

        return $valid;
    }<commit_msg>bugfix: analyzed tokenstream is only valid if position after analyzing is at end of tokenstream<commit_after><?php

/*
 * This file is part of the 'octris/core' package.
 *
 * (c) <NAME> <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Octris\Core\Parser;

/**
 * Class for defining a parser grammar.
 *
 * @copyright   copyright (c) 2014 by <NAME>
 * @author      <NAME> <<EMAIL>>
 */
class Grammar
{
    /**
     * Unknown token.
     *
     * @type    int
     */
    const T_UNKNOWN = 0;

    /**
     * ID of initial rule.
     *
     * @type    int|string|null
     */
    protected $initial = null;

    /**
     * Grammar rules.
     *
     * @type    array
     */
    protected $rules = [];

    /**
     * Events for tokens.
     *
     * @type    array
     */
    protected $events = [];

    /**
     * Registered tokens.
     *
     * @type    array
     */
    protected $tokens = array();

    /**
     * Constructor.
     */
    public function __construct()
    {
    }

    /**
     * Add a rule to the grammar.
     *
     * @param   int|string          $id                 Token identifier to apply the rule for.
     * @param   array               $rule               Grammar rule.
     * @param   bool                $initial            Whether to set the rule as initial.
     */
    public function addRule($id, array $rule, $initial = false)
    {
        if (isset($this->rules[$id])) {
            throw new \Exception('Rule is already defined!');
        }

        // { validate
        $iterator = new \RecursiveIteratorIterator(
            new \RecursiveArrayIterator($rule),
            true
        );

        foreach ($iterator as $k => $v) {
            if (!is_int($k)) {
                if (!is_array($v)) {
                    throw new \Exception(sprintf("No array specified for rule operator '%s'", $k));
                }

                switch ($k) {
                    case '$option':
                        if (($cnt = count($v)) != 1) {
                            throw new \Exception(sprintf("Rule operator '\$option' takes exactly one item, '%d' given", $cnt));
                        }
                        break;
                    case '$alternation':
                    case '$concatenation':
                    case '$repeat':
                        break;
                    default:
                        throw new \Exception(sprintf("Invalid rule operator '%s'", $k));
                }
            }
        }
        // }

        $this->rules[$id] = $rule;

        if ($initial) {
            $this->initial = $id;
        }
    }

    /**
     * Add an event for a token.
     *
     * @param   int                 $id                 Token identifier.
     * @param   callable            $cb                 Callback to call if the token occurs.
     */
    public function addEvent($id, callable $cb)
    {
        if (!isset($this->events[$id])) {
            $this->events[$id] = [];
        }

        $this->events[$id][] = $cb;
    }

    /**
     * Return list of defined tokens.
     *
     * @return  array                                   Defined tokens.
     */
    public function getTokens()
    {
        return $this->tokens;
    }

    /**
     * Return names of tokens. Will only work, if tokens are defined using class 'constants'.
     *
     * @return  array                                   Names of defined tokens.
     */
    public function getTokenNames()
    {
        return array_flip((new \ReflectionClass($this))->getConstants());
    }

    /**
     * Add a token to the registry.
     *
     * @param   string          $name                   Name of token.
     * @param   string          $regexp                 Regular expression for parser to match token.
     */
    public function addToken($name, $regexp)
    {
        $this->tokens[$name] = $regexp;
    }

    /**
     * Return the EBNF for the defined grammar.
     *
     * @return  string                                  The EBNF.
     */
    public function getEBNF()
    {
        $glue = array(
            '$concatenation' => array('', ' , ', ''),
            '$alternation'   => array('( ', ' | ', ' )'),
            '$repeat'        => array('{ ', '', ' }'),
            '$option'        => array('[ ', '', ' ]')
        );

        $render = function ($rule) use ($glue, &$render) {
            if (is_array($rule)) {
                $type = key($rule);

                foreach ($rule[$type] as &$_rule) {
                    $_rule = $render($_rule);
                }

                $return = $glue[$type][0] .
                          implode($glue[$type][1], $rule[$type]) .
                          $glue[$type][2];
            } else {
                $return = $rule;
            }

            return $return;
        };

        $return = '';

        foreach ($this->rules as $name => $rule) {
            $return .= $name . ' = ' . $render($rule) . " ;\n";
        }

        return $return;
    }

    /**
     * Analyze / validate token stream. If the token stream is invalid, the second, optional, parameter
     * will contain the expected token.
     *
     * @param   array               $tokens             Token stream to analyze.
     * @param   array               &$error             If an error occured the variable get's filled with the current token information and expected token(s).
     * @return  bool                                    Returns true if token stream is valid compared to the defined grammar.
     */
    public function analyze($tokens, &$error = null)
    {
        $expected = [];
        $pos      = 0;
        $error    = false;

        $v = function ($rule) use ($tokens, &$pos, &$v, &$expected, &$error) {
            $valid = false;

            if (is_scalar($rule) && isset($this->rules[$rule])) {
                // import rule
                $rule = $this->rules[$rule];
            }

            if (is_array($rule)) {
                $type = key($rule);

                switch ($type) {
                    case '$concatenation':
                        $state = $pos;

                        foreach ($rule[$type] as $_rule) {
                            if (!($valid = $v($_rule))) {
                                if ($error) {
                                    return false;
                                } elseif (($pos - $state) > 0) {
                                    $error = (isset($tokens[$pos])
                                              ? $tokens[$pos]
                                              : array_merge(
                                                  $tokens[$pos - 1],
                                                  array(
                                                      'token' => self::T_UNKNOWN,
                                                      'value' => self::T_UNKNOWN
                                                  )
                                              ));

                                    $error['expected'] = array_unique($expected);
                                    return false;
                                }
                                break;
                            }
                        }

                        if (!$valid) {
                            // rule did not match, restore position in token stream
                            $pos   = $state;
                            $valid = false;
                        }
                        break;
                    case '$alternation':
                        $state = $pos;

                        foreach ($rule[$type] as $_rule) {
                            if (($valid = $v($_rule)) || $error) {
                                // if ($error) return false;
                                break;
                            }
                        }

                        if (!$valid) {
                            // rule did not match, restore position in token stream
                            $pos   = $state;
                            $valid = false;
                        }
                        break;
                    case '$option':
                        $state = $pos;

                        foreach ($rule[$type] as $_rule) {
                            if (($valid = $v($_rule)) || $error) {
                                if ($error) {
                                    return false;
                                }
                                break;
                            }
                        }

                        if (!$valid) {
                            // rule did not match, restore position in token stream
                            $pos   = $state;
                            $valid = true;
                        }
                        break;
                    case '$repeat':
                        do {
                            $state = $pos;

                            foreach ($rule[$type] as $_rule) {
                                if (($valid = $v($_rule)) || $error) {
                                    if ($error) {
                                        return false;
                                    }
                                    break;
                                }
                            }
                        } while ($valid);

                        if (!$valid) {
                            // rule did not match, restore position in token stream
                            $pos   = $state;
                            $valid = true;
                        }
                        break;
                }
            } elseif (($valid = isset($tokens[$pos]))) {
                $token = $tokens[$pos];

                if (($valid = ($token['token'] == $rule))) {
                    ++$pos;
                    $expected = [];
                } else {
                    $expected[] = $rule;
                }
            } else {
                $expected[] = $rule;
            }

            return (!$error ? $valid : false);
        };

        if (!is_null($this->initial)) {
            $valid = $v($this->rules[$this->initial]);
        } else {
            // no initial rule, build one
            $valid = $v(['$alternation' => array_keys($this->rules)]);
        }

        $valid = (!$error && $valid && $pos == count($tokens));

        if ($valid) {
            foreach ($tokens as $token) {
                if (isset($this->events[$token['token']])) {
                    foreach ($this->events[$token['token']] as $event) {
                        $event($token);
                    }
                }
            }
        } elseif (!$error) {
            $error = $tokens[$pos];
            $error['expected'] = $expected;
        }

        return $valid;
    }
{'fraction_non_alphanumeric': 0.11881597062872877, 'fraction_numerical': 0.0011014226709499771, 'mean_word_length': 0.8610470578187719, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 74, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5489046', 'n_tokens_mistral': 5273, 'n_tokens_neox': 4990, 'n_words': 1354}