<commit_before><?php

/*
 * This file is part of the 'org.octris.core' package.
 *
 * (c) <NAME> <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace org\octris\core\parser {
    /**
     * Class for defining a parser grammar.
     *
     * @octdoc      c:parser/grammar
     * @copyright   copyright (c) 2014 by <NAME>
     * @author      <NAME> <<EMAIL>>
     */
    class grammar
    /**/
    {
        /**
         * Unknown token.
         *
         * @octdoc  d:grammar/T_UNKNOWN
         * @type    int
         */
        const T_UNKNOWN = 0;
        /**/
        
        /**
         * ID of initial rule.
         *
         * @octdoc  p:grammar/$initial
         * @type    int|string|null
         */
        protected $initial = null;
        /**/
        
        /**
         * Grammar rules.
         *
         * @octdoc  p:grammar/$rules
         * @type    array
         */
        protected $rules = [];
        /**/
        
        /**
         * Events for tokens.
         *
         * @octdoc  p:grammar/$events
         * @type    array
         */
        protected $events = [];
        /**/
        
        /**
         * Registered tokens.
         *
         * @octdoc  p:grammar/$tokens
         * @type    array
         */
        protected $tokens = array();
        /**/
        
        /**
         * Constructor.
         *
         * @octdoc  m:grammar/__construct
         */
        public function __construct()
        /**/
        {
        }
        
        /**
         * Add a rule to the grammar.
         *
         * @octdoc  m:grammar/addRule
         * @param   int|string          $id                 Token identifier to apply the rule for.
         * @param   array               $rule               Grammar rule.
         * @param   bool                $initial            Whether to set the rule as initial.
         */
        public function addRule($id, array $rule, $initial = false)
        /**/
        {
            if (isset($this->rules[$id])) {
                throw new \Exception('Rule is already defined!');
            }
            
            $this->rules[$id] = $rule;
            
            if ($initial) {
                $this->initial = $id;
            }
        }
        
        /**
         * Add an event for a token.
         *
         * @octdoc  m:grammar/addEvent
         * @param   int                 $id                 Token identifier.
         * @param   callable            $cb                 Callback to call if the token occurs.
         */
        public function addEvent($id, callable $cb)
        /**/
        {
            if (!isset($this->events[$id])) {
                $this->events[$id] = [];
            }
            
            $this->events[$id][] = $cb;
        }
        
        /**
         * Return list of defined tokens.
         *
         * @octdoc  m:grammar/getTokens
         * @return  array                                   Defined tokens.
         */
        public function getTokens()
        /**/
        {
            return $this->tokens;
        }
        
        /**
         * Add a token to the registry.
         *
         * @octdoc  m:grammar/addToken
         * @param   string          $name                   Name of token.
         * @param   string          $regexp                 Regular expression for parser to match token.
         */<commit_msg>added method to return token names<commit_after><?php

/*
 * This file is part of the 'org.octris.core' package.
 *
 * (c) <NAME> <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace org\octris\core\parser {
    /**
     * Class for defining a parser grammar.
     *
     * @octdoc      c:parser/grammar
     * @copyright   copyright (c) 2014 by <NAME>
     * @author      <NAME> <<EMAIL>>
     */
    class grammar
    /**/
    {
        /**
         * Unknown token.
         *
         * @octdoc  d:grammar/T_UNKNOWN
         * @type    int
         */
        const T_UNKNOWN = 0;
        /**/
        
        /**
         * ID of initial rule.
         *
         * @octdoc  p:grammar/$initial
         * @type    int|string|null
         */
        protected $initial = null;
        /**/
        
        /**
         * Grammar rules.
         *
         * @octdoc  p:grammar/$rules
         * @type    array
         */
        protected $rules = [];
        /**/
        
        /**
         * Events for tokens.
         *
         * @octdoc  p:grammar/$events
         * @type    array
         */
        protected $events = [];
        /**/
        
        /**
         * Registered tokens.
         *
         * @octdoc  p:grammar/$tokens
         * @type    array
         */
        protected $tokens = array();
        /**/
        
        /**
         * Constructor.
         *
         * @octdoc  m:grammar/__construct
         */
        public function __construct()
        /**/
        {
        }
        
        /**
         * Add a rule to the grammar.
         *
         * @octdoc  m:grammar/addRule
         * @param   int|string          $id                 Token identifier to apply the rule for.
         * @param   array               $rule               Grammar rule.
         * @param   bool                $initial            Whether to set the rule as initial.
         */
        public function addRule($id, array $rule, $initial = false)
        /**/
        {
            if (isset($this->rules[$id])) {
                throw new \Exception('Rule is already defined!');
            }
            
            $this->rules[$id] = $rule;
            
            if ($initial) {
                $this->initial = $id;
            }
        }
        
        /**
         * Add an event for a token.
         *
         * @octdoc  m:grammar/addEvent
         * @param   int                 $id                 Token identifier.
         * @param   callable            $cb                 Callback to call if the token occurs.
         */
        public function addEvent($id, callable $cb)
        /**/
        {
            if (!isset($this->events[$id])) {
                $this->events[$id] = [];
            }
            
            $this->events[$id][] = $cb;
        }
        
        /**
         * Return list of defined tokens.
         *
         * @octdoc  m:grammar/getTokens
         * @return  array                                   Defined tokens.
         */
        public function getTokens()
        /**/
        {
            return $this->tokens;
        }

        /**
         * Return names of tokens. Will only work, if tokens are defined using class 'constants'.
         *
         * @octdoc  m:grammar/getTokenNames
         * @return  array                                   Names of defined tokens.
         */
        public function getTokenNames()
        /**/
        {
            return array_flip((new \ReflectionClass($this))->getConstants());
        }
        
        /**
         * Add a token to the registry.
         *
         * @octdoc  m:grammar/addToken
         * @param   string          $name                   Name of token.
         * @param   string          $regexp                 Regular expression for parser to match token.
         */
{'fraction_non_alphanumeric': 0.11029115792298404, 'fraction_numerical': 0.001341741580571582, 'mean_word_length': 0.9600315540362871, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1980354', 'n_tokens_mistral': 1974, 'n_tokens_neox': 1951, 'n_words': 495}