<commit_before><?php

/*
 * This file is part of the JSON Lint package.
 *
 * (c) <NAME> <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Seld\JsonLint;

/**
 * Lexer class
 *
 * Ported from https://github.com/zaach/jsonlint
 */
class Lexer
{
    private $EOF = 1;
    private $rules = array(
        0 => '/^\s+/',
        1 => '/^-?([0-9]|[1-9][0-9]+)(\.[0-9]+)?([eE][+-]?[0-9]+)?\b/',
        2 => '{^"(\\\\["bfnrt/\\\\]|\\\\u[a-fA-F0-9]{4}|[^\0-\x09\x0a-\x1f\\\\"])*"}',
        3 => '/^\{/',
        4 => '/^\}/',
        5 => '/^\[/',
        6 => '/^\]/',
        7 => '/^,/',
        8 => '/^:/',
        9 => '/^true\b/',
        10 => '/^false\b/',
        11 => '/^null\b/',
        12 => '/^$/',
        13 => '/^./',
    );

    private $conditions = array(
        "INITIAL" => array(
            "rules" => array(0,1,2,3,4,5,6,7,8,9,10,11,12,13),
            "inclusive" => true,
        ),
    );

    public function lex()
    {
        $r = $this->next();
        if (!$r instanceof Undefined) {
            return $r;
        }
        return $this->lex();
    }

    public function setInput($input)
    {
        $this->_input = $input;
        $this->_more = $this->_less = $this->done = false;
        $this->yylineno = $this->yyleng = 0;
        $this->yytext = $this->matched = $this->match = '';
        $this->conditionStack = array('INITIAL');
        $this->yylloc = array('first_line' => 1, 'first_column' => 0, 'last_line' => 1, 'last_column' => 0);

        return $this;
    }

    public function showPosition()
    {
        $pre = $this->pastInput();
        $c = str_repeat('-', strlen($pre)); // new Array(pre.length + 1).join("-");
        return $pre . $this->upcomingInput() . "\n" . $c . "^";
    }

    protected function parseError($str, $hash)
    {
        throw new \Exception($str);
    }

    private function input()
    {
        $ch = $this->_input[0];
        $this->yytext += $ch;
        $this->yyleng++;
        $this->match += $ch;
        $this->matched += $ch;
        if (strpos($ch, "\n") !== false) {
            $this->yylineno++;
        }
        array_shift($this->_input); // slice(1)
        return $ch;
    }

    private function unput($ch)
    {
        $this->_input = $ch . $this->_input;
        return $this;
    }

    private function more()
    {
        $this->_more = true;
        return $this;
    }

    private function pastInput()
    {
        $past = substr($this->matched, 0, strlen($this->matched) - strlen($this->match));
        return (strlen($past) > 20 ? '...' : '') . str_replace("\n", '', substr($past, -20));
    }

    private function upcomingInput()
    {
        $next = $this->match;
        if (strlen($next) < 20) {
            $next += substr($this->_input, 0, 20 - strlen($next));
        }
        return str_replace("\n", '', substr($next, 0, 20) . (strlen($next) > 20 ? '...' : ''));<commit_msg>Remove dead code<commit_after><?php

/*
 * This file is part of the JSON Lint package.
 *
 * (c) <NAME> <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Seld\JsonLint;

/**
 * Lexer class
 *
 * Ported from https://github.com/zaach/jsonlint
 */
class Lexer
{
    private $EOF = 1;
    private $rules = array(
        0 => '/^\s+/',
        1 => '/^-?([0-9]|[1-9][0-9]+)(\.[0-9]+)?([eE][+-]?[0-9]+)?\b/',
        2 => '{^"(\\\\["bfnrt/\\\\]|\\\\u[a-fA-F0-9]{4}|[^\0-\x09\x0a-\x1f\\\\"])*"}',
        3 => '/^\{/',
        4 => '/^\}/',
        5 => '/^\[/',
        6 => '/^\]/',
        7 => '/^,/',
        8 => '/^:/',
        9 => '/^true\b/',
        10 => '/^false\b/',
        11 => '/^null\b/',
        12 => '/^$/',
        13 => '/^./',
    );

    private $conditions = array(
        "INITIAL" => array(
            "rules" => array(0,1,2,3,4,5,6,7,8,9,10,11,12,13),
            "inclusive" => true,
        ),
    );

    public function lex()
    {
        $r = $this->next();
        if (!$r instanceof Undefined) {
            return $r;
        }
        return $this->lex();
    }

    public function setInput($input)
    {
        $this->_input = $input;
        $this->_more = $this->_less = $this->done = false;
        $this->yylineno = $this->yyleng = 0;
        $this->yytext = $this->matched = $this->match = '';
        $this->conditionStack = array('INITIAL');
        $this->yylloc = array('first_line' => 1, 'first_column' => 0, 'last_line' => 1, 'last_column' => 0);

        return $this;
    }

    public function showPosition()
    {
        $pre = $this->pastInput();
        $c = str_repeat('-', strlen($pre)); // new Array(pre.length + 1).join("-");
        return $pre . $this->upcomingInput() . "\n" . $c . "^";
    }

    protected function parseError($str, $hash)
    {
        throw new \Exception($str);
    }

    private function pastInput()
    {
        $past = substr($this->matched, 0, strlen($this->matched) - strlen($this->match));
        return (strlen($past) > 20 ? '...' : '') . str_replace("\n", '', substr($past, -20));
    }

    private function upcomingInput()
    {
        $next = $this->match;
        if (strlen($next) < 20) {
            $next += substr($this->_input, 0, 20 - strlen($next));
        }
        return str_replace("\n", '', substr($next, 0, 20) . (strlen($next) > 20 ? '...' : ''));
{'fraction_non_alphanumeric': 0.23603504928806135, 'fraction_numerical': 0.028112449799196786, 'mean_word_length': 1.8140729327170004, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 105, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6584043', 'n_tokens_mistral': 1992, 'n_tokens_neox': 1903, 'n_words': 386}