<commit_before><?php

namespace Behat\Gherkin;

use Behat\Gherkin\Exception\ParserException,
    Behat\Gherkin\Node;

/*
 * This file is part of the Behat Gherkin.
 * (c) 2011 <NAME> <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

/**
 * Gherkin parser.
 *
 * $lexer  = new Behat\Gherkin\Lexer($keywords);
 * $parser = new Behat\Gherkin\Parser($lexer);
 * $featuresArray = $parser->parse('/path/to/feature.feature');
 *
 * @author      <NAME> <<EMAIL>>
 */
class Parser
{
    private $file;
    private $lexer;

    /**
     * Initializes parser.
     *
     * @param   Behat\Gherkin\Lexer     $lexer  lexer instance
     */
    public function __construct(Lexer $lexer)
    {
        $this->lexer = $lexer;
    }

    /**
     * Parses input & returns features array.
     *
     * @param   string          $input  gherkin filename or string document
     *
     * @return  array                   array of feature nodes
     *
     * @throws  Behat\Gherkin\Exception\ParserException if feature file has more than one language specifier
     */
    public function parse($input, $file = null)
    {
        $features = array();

        if (is_file($input)) {
            $this->file = $input;
            $input      = file_get_contents($this->file);
        } else {
            $this->file = $file;
        }

        $this->lexer->setInput($input);
        $this->lexer->setLanguage($language = 'en');
        $languageSpecifierLine = null;

        while ('EOS' !== ($predicted = $this->predictTokenType())) {
            if ('Newline' === $predicted || 'Comment' === $predicted) {
                $this->lexer->getAdvancedToken();
            } elseif ('Language' === $predicted) {
                $token      = $this->expectTokenType('Language');
                $language   = $token->value;

                if (null === $languageSpecifierLine) {
                    // Reparse input with new language
                    $languageSpecifierLine = $token->line;
                    $this->lexer->setInput($input);
                    $this->lexer->setLanguage($language);
                } elseif ($languageSpecifierLine !== $token->line) {
                    // Language already specified
                    throw new ParserException(sprintf('Ambigious language specifiers on lines: %d and %d%s',
                        $languageSpecifierLine, $token->line,
                        $this->file ? ' in file: ' . $this->file : ''
                    ));
                }
            } elseif ('Feature' === $predicted
                   || ('Tag' === $predicted && 'Feature' === $this->predictTokenType(2))) {
                $feature = $this->parseExpression();
                $feature->setLanguage($language);
                $features[] = $feature;
            } else {
                $this->expectTokenType(array('Feature', 'Scenario', 'Outline'));
            }
        }

        return $features;
    }

    /**
     * Returns next token if it's type equals to expected.
     *
     * @param   string|array    $types  expected type or types of the next token
     *
     * @return  stdClass
     *
     * @throws  Behat\Gherkin\Exception\ParserException if token type is differ from expected one
     */
    protected function expectTokenType($types)
    {
        $types = (array) $types;

        foreach ($types as $type) {
            if ($type === $this->predictTokenType()) {
                return $this->lexer->getAdvancedToken();
            }
        }

        throw new ParserException(sprintf('Expected %s %s, but got %s on line: %d%s',
            implode(', ', $types), (count($types) > 1 ? 'tokens' : 'token'), $this->predictTokenType(),
            $this->lexer->predictToken()->line, $this->file ? ' in file: ' . $this->file : ''
        ));
    }

    /**
     * Returns next token if it's type equals to expected.
     *
     * @param   string  $type   type
     *
     * @return  stdClass
     */
    protected function acceptTokenType($type)
    {
        if ($type === $this->predictTokenType()) {
            return $this->lexer->getAdvancedToken();
        }
    }<commit_msg>rolled back old Parser::expectTokenType behavior<commit_after><?php

namespace Behat\Gherkin;

use Behat\Gherkin\Exception\ParserException,
    Behat\Gherkin\Node;

/*
 * This file is part of the Behat Gherkin.
 * (c) 2011 <NAME> <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

/**
 * Gherkin parser.
 *
 * $lexer  = new Behat\Gherkin\Lexer($keywords);
 * $parser = new Behat\Gherkin\Parser($lexer);
 * $featuresArray = $parser->parse('/path/to/feature.feature');
 *
 * @author      <NAME> <<EMAIL>>
 */
class Parser
{
    private $file;
    private $lexer;

    /**
     * Initializes parser.
     *
     * @param   Behat\Gherkin\Lexer     $lexer  lexer instance
     */
    public function __construct(Lexer $lexer)
    {
        $this->lexer = $lexer;
    }

    /**
     * Parses input & returns features array.
     *
     * @param   string          $input  gherkin filename or string document
     *
     * @return  array                   array of feature nodes
     *
     * @throws  Behat\Gherkin\Exception\ParserException if feature file has more than one language specifier
     */
    public function parse($input, $file = null)
    {
        $features = array();

        if (is_file($input)) {
            $this->file = $input;
            $input      = file_get_contents($this->file);
        } else {
            $this->file = $file;
        }

        $this->lexer->setInput($input);
        $this->lexer->setLanguage($language = 'en');
        $languageSpecifierLine = null;

        while ('EOS' !== ($predicted = $this->predictTokenType())) {
            if ('Newline' === $predicted || 'Comment' === $predicted) {
                $this->lexer->getAdvancedToken();
            } elseif ('Language' === $predicted) {
                $token      = $this->expectTokenType('Language');
                $language   = $token->value;

                if (null === $languageSpecifierLine) {
                    // Reparse input with new language
                    $languageSpecifierLine = $token->line;
                    $this->lexer->setInput($input);
                    $this->lexer->setLanguage($language);
                } elseif ($languageSpecifierLine !== $token->line) {
                    // Language already specified
                    throw new ParserException(sprintf('Ambigious language specifiers on lines: %d and %d%s',
                        $languageSpecifierLine, $token->line,
                        $this->file ? ' in file: ' . $this->file : ''
                    ));
                }
            } elseif ('Feature' === $predicted
                   || ('Tag' === $predicted && 'Feature' === $this->predictTokenType(2))) {
                $feature = $this->parseExpression();
                $feature->setLanguage($language);
                $features[] = $feature;
            } else {
                $this->expectTokenType('Feature');
            }
        }

        return $features;
    }

    /**
     * Returns next token if it's type equals to expected.
     *
     * @param   string  $types  expected type
     *
     * @return  stdClass
     *
     * @throws  Behat\Gherkin\Exception\ParserException if token type is differ from expected one
     */
    protected function expectTokenType($type)
    {
        if ($type === $this->predictTokenType()) {
            return $this->lexer->getAdvancedToken();
        }

        throw new ParserException(sprintf('Expected %s token, but got %s on line: %d%s',
            $type, $this->predictTokenType(), $this->lexer->predictToken()->line,
            $this->file ? ' in file: ' . $this->file : ''
        ));
    }

    /**
     * Returns next token if it's type equals to expected.
     *
     * @param   string  $type   type
     *
     * @return  stdClass
     */
    protected function acceptTokenType($type)
    {
        if ($type === $this->predictTokenType()) {
            return $this->lexer->getAdvancedToken();
        }
    }
{'fraction_non_alphanumeric': 0.1379268588035844, 'fraction_numerical': 0.001332041656575442, 'mean_word_length': 1.8332761578044596, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 94, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2334485', 'n_tokens_mistral': 2366, 'n_tokens_neox': 2281, 'n_words': 548}