<commit_before><?php declare(strict_types=1);

namespace CalendarBundle\Lexer;

/**
 * iCal-Tcl Lexer.
 *
 * The underlying interface for this class has been extracted from the ical-3.0.2 source.
 * See URL below.
 *
 * @source https://fossies.org/dox/ical-3.0.2/lexer_8h_source.html (accessed 30th September 2016)
 *
 * @package CalendarBundle\Lexer
 * @author <NAME> <<EMAIL>>
 */
class ICalLexer implements LexerInterface
{
    const OPEN_STRING = "[";
    const CLOSE_STRING = "]";

    /**
     * Entire File Contents
     *
     * @var string
     */
    protected $buf;

    /**
     * self::$buf length
     *
     * @var int
     */
    protected $length;

    /**
     * Index of next character in file
     *
     * @var int
     */
    protected $index;

    /**
     * Lexer constructor.
     *
     * @param string $buffer
     */
    public function __construct(string $buffer)
    {
        $this->buf = $buffer;
        $this->length = strlen($this->buf);
        $this->index = 0;
    }

    /**
     * Get the current status
     *
     * @return int
     */
    public function status(): int
    {
        if ($this->index === $this->length) {
            return self::EOF;
        } else if ($this->index > $this->length) {
            return self::ERROR;
        } else {
            return self::VALID;
        }
    }

    /**
     * Return next character without advancing
     *
     * @return string
     */
    public function peek(): string
    {
        if ($this->index < $this->length) {
            return $this->buf[$this->index];
        }

        return "";
    }

    /**
     * Return next character and advance over it
     *
     * @return string
     */
    public function next(): string
    {
        if ($this->index < $this->length) {
            return $this->buf[$this->index++];
        }

        return "";
    }

    /**
     * Advance and then peek()
     *
     * @return string
     */
    public function advance(): string
    {
        $this->index++;

        if ($this->index < $this->length) {
            return $this->buf[$this->index];
        }

        // otherwise, undo the advance
        $this->index--;

        return "";
    }

    /**
     * Move position over expected text (character only)
     *
     * @param string $char
     *
     * @throws LexerException
     */
    public function skip(string $char)
    {
        if (strlen($char) > 1) {
            throw new LexerException("can only skip over characters");
        }

        if (($this->index < $this->length) && ($this->buf[$this->index] === $char)) {
            $this->index++;
            return;
        }

        throw new LexerException("unexpected character: " . $char);
    }

    /**
     * Skip all whitespace
     *
     * @return void
     */
    public function skipWhitespace()
    {
        while ($this->index < $this->length) {
            $char = $this->buf[$this->index];

            if (!ctype_space($char)) {
                return;
            }

            $this->index++;
        }
    }

    /**
     * Read an identifier.
     *
     * An identifier is a non-empty sequence of alphanumeric characters
     * (including underscores), where the first character is not a digit.
     *
     * @throws LexerException
     *
     * @return string
     */
    public function getId(): string
    {
        if ($this->index >= $this->length) {
            return "";
        }

        if (!$this->isLetter($this->buf[$this->index])) {
            throw new LexerException("illegal character, was expecting ID");
        }

        $begin = $this->index;

        while (($this->index < $this->length) && $this->isNumOrLetter($this->buf[$this->index])) {
            $this->index++;
        }

        return substr($this->buf, $begin, $this->index - $begin);
    }

    /**
     * Is a given char (or string) a letter?
     *<commit_msg>#3: skip whitespace in getId() of ICalLexer<commit_after><?php declare(strict_types=1);

namespace CalendarBundle\Lexer;

/**
 * iCal-Tcl Lexer.
 *
 * The underlying interface for this class has been extracted from the ical-3.0.2 source.
 * See URL below.
 *
 * @source https://fossies.org/dox/ical-3.0.2/lexer_8h_source.html (accessed 30th September 2016)
 *
 * @package CalendarBundle\Lexer
 * @author <NAME> <<EMAIL>>
 */
class ICalLexer implements LexerInterface
{
    const OPEN_STRING = "[";
    const CLOSE_STRING = "]";

    /**
     * Entire File Contents
     *
     * @var string
     */
    protected $buf;

    /**
     * self::$buf length
     *
     * @var int
     */
    protected $length;

    /**
     * Index of next character in file
     *
     * @var int
     */
    protected $index;

    /**
     * Lexer constructor.
     *
     * @param string $buffer
     */
    public function __construct(string $buffer)
    {
        $this->buf = $buffer;
        $this->length = strlen($this->buf);
        $this->index = 0;
    }

    /**
     * Get the current status
     *
     * @return int
     */
    public function status(): int
    {
        if ($this->index === $this->length) {
            return self::EOF;
        } else if ($this->index > $this->length) {
            return self::ERROR;
        } else {
            return self::VALID;
        }
    }

    /**
     * Return next character without advancing
     *
     * @return string
     */
    public function peek(): string
    {
        if ($this->index < $this->length) {
            return $this->buf[$this->index];
        }

        return "";
    }

    /**
     * Return next character and advance over it
     *
     * @return string
     */
    public function next(): string
    {
        if ($this->index < $this->length) {
            return $this->buf[$this->index++];
        }

        return "";
    }

    /**
     * Advance and then peek()
     *
     * @return string
     */
    public function advance(): string
    {
        $this->index++;

        if ($this->index < $this->length) {
            return $this->buf[$this->index];
        }

        // otherwise, undo the advance
        $this->index--;

        return "";
    }

    /**
     * Move position over expected text (character only)
     *
     * @param string $char
     *
     * @throws LexerException
     */
    public function skip(string $char)
    {
        if (strlen($char) > 1) {
            throw new LexerException("can only skip over characters");
        }

        if (($this->index < $this->length) && ($this->buf[$this->index] === $char)) {
            $this->index++;
            return;
        }

        throw new LexerException("unexpected character: " . $char);
    }

    /**
     * Skip all whitespace
     *
     * @return void
     */
    public function skipWhitespace()
    {
        while ($this->index < $this->length) {
            $char = $this->buf[$this->index];

            if (!ctype_space($char)) {
                return;
            }

            $this->index++;
        }
    }

    /**
     * Read an identifier.
     *
     * An identifier is a non-empty sequence of alphanumeric characters
     * (including underscores), where the first character is not a digit.
     *
     * @throws LexerException
     *
     * @return string
     */
    public function getId(): string
    {
        if ($this->index >= $this->length) {
            return "";
        }

        // identifiers cannot be whitespace, so we can safely skip it.
        $this->skipWhitespace();

        if (!$this->isLetter($this->buf[$this->index])) {
            throw new LexerException("illegal character, was expecting ID");
        }

        $begin = $this->index;

        while (($this->index < $this->length) && $this->isNumOrLetter($this->buf[$this->index])) {
            $this->index++;
        }

        return substr($this->buf, $begin, $this->index - $begin);
    }

    /**
     * Is a given char (or string) a letter?
     *
{'fraction_non_alphanumeric': 0.146046159667045, 'fraction_numerical': 0.004161937192584185, 'mean_word_length': 1.6557267247153382, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 108, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3090670', 'n_tokens_mistral': 2394, 'n_tokens_neox': 2300, 'n_words': 576}