<commit_before><?php
/**
 * The base tokenizer class.
 *
 * @author    <NAME> <<EMAIL>>
 * @copyright 2006-2015 Squiz Pty Ltd (ABN 77 084 670 600)
 * @license   https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
 */

namespace PHP_CodeSniffer\Tokenizers;

use PHP_CodeSniffer\Exceptions\RuntimeException;
use PHP_CodeSniffer\Util;

abstract class Tokenizer
{

    /**
     * The config data for the run.
     *
     * @var \PHP_CodeSniffer\Config
     */
    protected $config = null;

    /**
     * The EOL char used in the content.
     *
     * @var string
     */
    protected $eolChar = [];

    /**
     * A token-based representation of the content.
     *
     * @var array
     */
    protected $tokens = [];

    /**
     * Known lengths of tokens.
     *
     * @var array<int, int>
     */
    public $knownLengths = [];

    /**
     * A list of lines being ignored due to error suppression comments.
     *
     * @var array
     */
    public $ignoredLines = [];


    /**
     * Initialise and run the tokenizer.
     *
     * @param string                         $content The content to tokenize,
     * @param \PHP_CodeSniffer\Config | null $config  The config data for the run.
     * @param string                         $eolChar The EOL char used in the content.
     *
     * @return void
     * @throws TokenizerException If the file appears to be minified.
     */
    public function __construct($content, $config, $eolChar='\n')
    {
        $this->eolChar = $eolChar;

        $this->config = $config;
        $this->tokens = $this->tokenize($content);

        if ($config === null) {
            return;
        }

        $this->createPositionMap();
        $this->createTokenMap();
        $this->createParenthesisNestingMap();
        $this->createScopeMap();
        $this->createLevelMap();

        // Allow the tokenizer to do additional processing if required.
        $this->processAdditional();

    }//end __construct()


    /**
     * Checks the content to see if it looks minified.
     *
     * @param string $content The content to tokenize,
     * @param string $eolChar The EOL char used in the content.
     *
     * @return boolean
     */
    protected function isMinifiedContent($content, $eolChar='\n')
    {
        // Minified files often have a very large number of characters per line
        // and cause issues when tokenizing.
        $numChars = strlen($content);
        $numLines = (substr_count($content, $eolChar) + 1);
        $average  = ($numChars / $numLines);
        if ($average > 100) {
            return true;
        }

        return false;

    }//end isMinifiedContent()


    /**
     * Gets the array of tokens.
     *
     * @return array
     */
    public function getTokens()
    {
        return $this->tokens;

    }//end getTokens()


    /**
     * Creates an array of tokens when given some content.
     *
     * @param string $string The string to tokenize.
     *
     * @return array
     */
    abstract protected function tokenize($string);


    /**
     * Performs additional processing after main tokenizing.
     *
     * @return void
     */
    abstract protected function processAdditional();


    /**
     * Sets token position information.
     *
     * Can also convert tabs into spaces. Each tab can represent between
     * 1 and $width spaces, so this cannot be a straight string replace.
     *
     * @return void
     */
    private function createPositionMap()
    {
        $currColumn = 1;
        $lineNumber = 1;
        $eolLen     = strlen($this->eolChar);
        $ignoring   = null;
        $inTests    = defined('PHP_CODESNIFFER_IN_TESTS');

        $checkEncoding = false;
        if (function_exists('iconv_strlen') === true) {
            $checkEncoding = true;
        }

        $checkAnnotations = $this->config->annotations;

        $this->tokensWithTabs = [
            T_WHITESPACE               => true,
            T_COMMENT                  => true,
            T_DOC_COMMENT              => true,
            T_DOC_COMMENT_WHITESPACE   => true,
            T_DOC_COMMENT_STRING       => true,
            T_CONSTANT_ENCAPSED_STRING => true,
            T_DOUBLE_QUOTED_STRING     => true,
            T_HEREDOC                  => true,
            T_NOWDOC                   => true,
            T_INLINE_HTML              => true,
        ];

        $this->numTokens = count($this->tokens);
        for ($i = 0; $i < $this->numTokens; $i++) {
            $this->tokens[$i]['line']   = $lineNumber;
            $this->tokens[$i]['column'] = $currColumn;

            if (isset($this->knownLengths[$this->tokens[$i]['code']]) === true) {
                // There are no tabs in the tokens we know the length of.
                $length      = $this->knownLengths[$this->tokens[$i]['code']];
                $currColumn += $length;
            } else if ($this->config->tabWidth === 0
                || isset($this->tokensWithTabs[$this->tokens[$i]['code']]) === false
                || strpos($this->tokens[$i]['content'], "\t") === false
            ) {
                // There are no tabs in this content, or we aren't replacing them.
                if ($checkEncoding === true) {
                    // Not using the default encoding, so take a bit more care.
                    $oldLevel = error_reporting();
                    error_reporting(0);
                    $length = iconv_strlen($this->tokens[$i]['content'], $this->config->encoding);
                    error_reporting($oldLevel);

                    if ($length === false) {
                        // String contained invalid characters, so revert to default.
                        $length = strlen($this->tokens[$i]['content']);
                    }
                } else {
                    $length = strlen($this->tokens[$i]['content']);
                }

                $currColumn += $length;
            } else {
                $this->replaceTabsInToken($this->tokens[$i]);
                $length      = $this->tokens[$i]['length'];
                $currColumn += $length;
            }//end if

            $this->tokens[$i]['length'] = $length;

            if (isset($this->knownLengths[$this->tokens[$i]['code']]) === false
                && strpos($this->tokens[$i]['content'], $this->eolChar) !== false
            ) {
                $lineNumber++;
                $currColumn = 1;

                // Newline chars are not counted in the token length.
                $this->tokens[$i]['length'] -= $eolLen;
            }

            if ($checkAnnotations === true
                && ($this->tokens[$i]['code'] === T_COMMENT
                || $this->tokens[$i]['code'] === T_DOC_COMMENT_STRING
                || $this->tokens[$i]['code'] === T_DOC_COMMENT_TAG
                || ($inTests === true && $this->tokens[$i]['code'] === T_INLINE_HTML))
            ) {
                $commentText      = ltrim($this->tokens[$i]['content'], " \t/*");
                $commentText      = rtrim($commentText, " */\t\r\n");
                $commentTextLower = strtolower($commentText);
                if (strpos($commentText, '@codingStandards') !== false) {
                    // If this comment is the only thing on the line, it tells us
                    // to ignore the following line. If the line contains other content
                    // then we are just ignoring this one single line.
                    $ownLine = false;
                    if ($i > 0) {
                        for ($prev = ($i - 1); $prev >= 0; $prev--) {
                            if ($this->tokens[$prev]['code'] === T_WHITESPACE) {
                                continue;
                            }

                            break;
                        }

                        if ($this->tokens[$prev]['line'] !== $this->tokens[$i]['line']) {
                            $ownLine = true;
                        }
                    }

                    if ($ignoring === null
                        && strpos($commentText, '@codingStandardsIgnoreStart') !== false
                    ) {
                        $ignoring = ['all' => true];
                        if ($ownLine === true) {
                            $this->ignoredLines[$this->tokens[$i]['line']] = $ignoring;
                        }
                    } else if ($ignoring !== null
                        && strpos($commentText, '@codingStandardsIgnoreEnd') !== false
                    ) {
                        if ($ownLine === true) {
                            $this->ignoredLines[$this->tokens[$i]['line']] = ['all' => true];
                        } else {
                            $this->ignoredLines[$this->tokens[$i]['line']] = $ignoring;
                        }

                        $ignoring = null;
                    } else if ($ignoring === null
                        && strpos($commentText, '@codingStandardsIgnoreLine') !== false
                    ) {
                        $ignoring = ['all' => true];
                        if ($ownLine === true) {
                            $this->ignoredLines[$this->tokens[$i]['line']]       = $ignoring;
                            $this->ignoredLines[($this->tokens[$i]['line'] + 1)] = $ignoring;
                        } else {
                            $this->ignoredLines[$this->tokens[$i]['line']] = $ignoring;
                        }

                        $ignoring = null;
                    }//end if
                } else if (substr($commentTextLower, 0, 6) === 'phpcs:'
                    || substr($commentTextLower, 0, 7) === '@phpcs:'
                ) {
                    // If the @phpcs: syntax is being used, strip the @ to make
                    // comparisions easier.
                    if ($commentText[0] === '@') {
                        $commentText      = substr($commentText, 1);
                        $commentTextLower = strtolower($commentText);
                    }

                    // If there is a comment on the end, strip it off.
                    $commentStart = strpos($commentTextLower, ' --');
                    if ($commentStart !== false) {
                        $commentText      = substr($commentText, 0, $commentStart);
                        $commentTextLower = strtolower($commentText);
                    }

                    // If this comment is the only thing on the line, it tells us
                    // to ignore the following line. If the line contains other content
                    // then we are just ignoring this one single line.
                    $ownLine = false;
                    if ($i > 0) {
                        for ($prev = ($i - 1); $prev > 0; $prev--) {
                            if ($this->tokens[$prev]['code'] === T_WHITESPACE
                                || $this->tokens[$prev]['code'] === T_OPEN_TAG
                                || ($this->tokens[$prev]['code'] === T_INLINE_HTML
                                && trim($this->tokens[$prev]['content']) === '')
                            ) {
                                continue;
                            }

                            break;
                        }

                        if ($this->tokens[$prev]['line'] !== $this->tokens[$i]['line']) {
                            $ownLine = true;
                        }
                    }

                    if (substr($commentTextLower, 0, 9) === 'phpcs:set') {
                        // Ignore standards for complete lines that change sniff settings.
                        if ($ownLine === true) {
                            $this->ignoredLines[$this->tokens[$i]['line']] = true;
                        }

                        $this->tokens[$i]['code'] = T_PHPCS_SET;
                        $this->tokens[$i]['type'] = 'T_PHPCS_SET';
                    } else if (substr($commentTextLower, 0, 16) === 'phpcs:ignorefile') {
                        // The whole file will be ignored, but at least set the correct token.
                        $this->tokens[$i]['code'] = T_PHPCS_IGNORE_FILE;
                        $this->tokens[$i]['type'] = 'T_PHPCS_IGNORE_FILE';
                    } else if (substr($commentTextLower, 0, 13) === 'phpcs:disable') {
                        if ($ownLine === true) {
                            // Completely ignore the comment line.
                            $this->ignoredLines[$this->tokens[$i]['line']] = ['all' => true];
                        }

                        if ($ignoring === null) {
                            $ignoring = [];
                        }

                        $disabledSniffs = [];

                        $additionalText = substr($commentText, 14);
                        if ($additionalText === false) {
                            $ignoring = ['all' => true];
                        } else {
                            $parts = explode(',', substr($commentText, 13));
                            foreach ($parts as $sniffCode) {
                                $sniffCode = trim($sniffCode);
                                $disabledSniffs[$sniffCode] = true;
                                $ignoring[$sniffCode]       = true;
                            }
                        }

                        $this->tokens[$i]['code']       = T_PHPCS_DISABLE;
                        $this->tokens[$i]['type']       = 'T_PHPCS_DISABLE';
                        $this->tokens[$i]['sniffCodes'] = $disabledSniffs;
                    } else if (substr($commentTextLower, 0, 12) === 'phpcs:enable') {
                        if ($ignoring !== null) {
                            $enabledSniffs = [];

                            $additionalText = substr($commentText, 13);
                            if ($additionalText === false) {
                                $ignoring = null;
                            } else {
                                $parts = explode(',', substr($commentText, 13));
                                foreach ($parts as $sniffCode) {
                                    $sniffCode = trim($sniffCode);
                                    $enabledSniffs[$sniffCode] = true;
                                    if (isset($ignoring[$sniffCode]) === true) {
                                        unset($ignoring[$sniffCode]);
                                    }
                                }

                                if (empty($ignoring) === true) {
                                    $ignoring = null;
                                }
                            }

                            if ($ownLine === true) {
                                // Completely ignore the comment line.
                                $this->ignoredLines[$this->tokens[$i]['line']] = ['all' => true];
                            } else {
                                // The comment is on the same line as the code it is ignoring,
                                // so respect the new ignore rules.
                                $this->ignoredLines[$this->tokens[$i]['line']] = $ignoring;
                            }

                            $this->tokens[$i]['sniffCodes'] = $enabledSniffs;
                        }//end if

                        $this->tokens[$i]['code'] = T_PHPCS_ENABLE;
                        $this->tokens[$i]['type'] = 'T_PHPCS_ENABLE';
                    } else if (substr($commentTextLower, 0, 12) === 'phpcs:ignore') {
                        $ignoreRules = [];

                        $additionalText = substr($commentText, 13);
                        if ($additionalText === false) {
                            $ignoreRules = ['all' => true];
                        } else {
                            $parts = explode(',', substr($commentText, 13));
                            foreach ($parts as $sniffCode) {
                                $ignoreRules[trim($sniffCode)] = true;
                            }
                        }

                        $this->tokens[$i]['code']       = T_PHPCS_IGNORE;
                        $this->tokens[$i]['type']       = 'T_PHPCS_IGNORE';
                        $this->tokens[$i]['sniffCodes'] = $ignoreRules;

                        if ($ignoring !== null) {
                            $ignoreRules += $ignoring;
                        }

                        if ($ownLine === true) {
                            // Completely ignore the comment line, and set the folllowing
                            // line to include the ignore rules we've set.
                            $this->ignoredLines[$this->tokens[$i]['line']]       = ['all' => true];
                            $this->ignoredLines[($this->tokens[$i]['line'] + 1)] = $ignoreRules;
                        } else {
                            // The comment is on the same line as the code it is ignoring,
                            // so respect the ignore rules it set.
                            $this->ignoredLines[$this->tokens[$i]['line']] = $ignoreRules;
                        }
                    }//end if
                }//end if
            }//end if

            if ($ignoring !== null && isset($this->ignoredLines[$this->tokens[$i]['line']]) === false) {
                $this->ignoredLines[$this->tokens[$i]['line']] = $ignoring;
            }
        }//end for

    }//end createPositionMap()


    /**
     * Replaces tabs in original token content with spaces.
     *
     * Each tab can represent between 1 and $config->tabWidth spaces,
     * so this cannot be a straight string replace. The original content
     * is placed into an orig_content index and the new token length is also
     * set in the length index.
     *
     * @param array  $token    The token to replace tabs inside.
     * @param string $prefix   The character to use to represent the start of a tab.
     * @param string $padding  The character to use to represent the end of a tab.
     * @param int    $tabWidth The number of spaces each tab represents.
     *
     * @return void
     */
    public function replaceTabsInToken(&$token, $prefix=' ', $padding=' ', $tabWidth=null)
    {
        $checkEncoding = false;
        if (function_exists('iconv_strlen') === true) {
            $checkEncoding = true;
        }

        $currColumn = $token['column'];
        if ($tabWidth === null) {
            $tabWidth = $this->config->tabWidth;
            if ($tabWidth === 0) {
                $tabWidth = 1;
            }
        }

        if (str_replace("\t", '', $token['content']) === '') {
            // String only contains tabs, so we can shortcut the process.
            $numTabs = strlen($token['content']);

            $firstTabSize = ($tabWidth - (($currColumn - 1) % $tabWidth));
            $length       = ($firstTabSize + ($tabWidth * ($numTabs - 1)));
            $newContent   = $prefix.str_repeat($padding, ($length - 1));
        } else {
            // We need to determine the length of each tab.
            $tabs = explode("\t", $token['content']);

            $numTabs    = (count($tabs) - 1);
            $tabNum     = 0;
            $newContent = '';
            $length     = 0;

            foreach ($tabs as $content) {
                if ($content !== '') {
                    $newContent .= $content;
                    if ($checkEncoding === true) {
                        // Not using the default encoding, so take a bit more care.
                        $oldLevel = error_reporting();
                        error_reporting(0);
                        $contentLength = iconv_strlen($content, $this->config->encoding);
                        error_reporting($oldLevel);
                        if ($contentLength === false) {
                            // String contained invalid characters, so revert to default.
                            $contentLength = strlen($content);
                        }<commit_msg>Improve Tokenizer::replaceTabsInToken performance<commit_after><?php
/**
 * The base tokenizer class.
 *
 * @author    <NAME> <<EMAIL>>
 * @copyright 2006-2015 Squiz Pty Ltd (ABN 77 084 670 600)
 * @license   https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
 */

namespace PHP_CodeSniffer\Tokenizers;

use PHP_CodeSniffer\Exceptions\RuntimeException;
use PHP_CodeSniffer\Util;

abstract class Tokenizer
{

    /**
     * The config data for the run.
     *
     * @var \PHP_CodeSniffer\Config
     */
    protected $config = null;

    /**
     * The EOL char used in the content.
     *
     * @var string
     */
    protected $eolChar = [];

    /**
     * A token-based representation of the content.
     *
     * @var array
     */
    protected $tokens = [];

    /**
     * Known lengths of tokens.
     *
     * @var array<int, int>
     */
    public $knownLengths = [];

    /**
     * A list of lines being ignored due to error suppression comments.
     *
     * @var array
     */
    public $ignoredLines = [];


    /**
     * Initialise and run the tokenizer.
     *
     * @param string                         $content The content to tokenize,
     * @param \PHP_CodeSniffer\Config | null $config  The config data for the run.
     * @param string                         $eolChar The EOL char used in the content.
     *
     * @return void
     * @throws TokenizerException If the file appears to be minified.
     */
    public function __construct($content, $config, $eolChar='\n')
    {
        $this->eolChar = $eolChar;

        $this->config = $config;
        $this->tokens = $this->tokenize($content);

        if ($config === null) {
            return;
        }

        $this->createPositionMap();
        $this->createTokenMap();
        $this->createParenthesisNestingMap();
        $this->createScopeMap();
        $this->createLevelMap();

        // Allow the tokenizer to do additional processing if required.
        $this->processAdditional();

    }//end __construct()


    /**
     * Checks the content to see if it looks minified.
     *
     * @param string $content The content to tokenize,
     * @param string $eolChar The EOL char used in the content.
     *
     * @return boolean
     */
    protected function isMinifiedContent($content, $eolChar='\n')
    {
        // Minified files often have a very large number of characters per line
        // and cause issues when tokenizing.
        $numChars = strlen($content);
        $numLines = (substr_count($content, $eolChar) + 1);
        $average  = ($numChars / $numLines);
        if ($average > 100) {
            return true;
        }

        return false;

    }//end isMinifiedContent()


    /**
     * Gets the array of tokens.
     *
     * @return array
     */
    public function getTokens()
    {
        return $this->tokens;

    }//end getTokens()


    /**
     * Creates an array of tokens when given some content.
     *
     * @param string $string The string to tokenize.
     *
     * @return array
     */
    abstract protected function tokenize($string);


    /**
     * Performs additional processing after main tokenizing.
     *
     * @return void
     */
    abstract protected function processAdditional();


    /**
     * Sets token position information.
     *
     * Can also convert tabs into spaces. Each tab can represent between
     * 1 and $width spaces, so this cannot be a straight string replace.
     *
     * @return void
     */
    private function createPositionMap()
    {
        $currColumn = 1;
        $lineNumber = 1;
        $eolLen     = strlen($this->eolChar);
        $ignoring   = null;
        $inTests    = defined('PHP_CODESNIFFER_IN_TESTS');

        $checkEncoding = false;
        if (function_exists('iconv_strlen') === true) {
            $checkEncoding = true;
        }

        $checkAnnotations = $this->config->annotations;

        $this->tokensWithTabs = [
            T_WHITESPACE               => true,
            T_COMMENT                  => true,
            T_DOC_COMMENT              => true,
            T_DOC_COMMENT_WHITESPACE   => true,
            T_DOC_COMMENT_STRING       => true,
            T_CONSTANT_ENCAPSED_STRING => true,
            T_DOUBLE_QUOTED_STRING     => true,
            T_HEREDOC                  => true,
            T_NOWDOC                   => true,
            T_INLINE_HTML              => true,
        ];

        $this->numTokens = count($this->tokens);
        for ($i = 0; $i < $this->numTokens; $i++) {
            $this->tokens[$i]['line']   = $lineNumber;
            $this->tokens[$i]['column'] = $currColumn;

            if (isset($this->knownLengths[$this->tokens[$i]['code']]) === true) {
                // There are no tabs in the tokens we know the length of.
                $length      = $this->knownLengths[$this->tokens[$i]['code']];
                $currColumn += $length;
            } else if ($this->config->tabWidth === 0
                || isset($this->tokensWithTabs[$this->tokens[$i]['code']]) === false
                || strpos($this->tokens[$i]['content'], "\t") === false
            ) {
                // There are no tabs in this content, or we aren't replacing them.
                if ($checkEncoding === true) {
                    // Not using the default encoding, so take a bit more care.
                    $oldLevel = error_reporting();
                    error_reporting(0);
                    $length = iconv_strlen($this->tokens[$i]['content'], $this->config->encoding);
                    error_reporting($oldLevel);

                    if ($length === false) {
                        // String contained invalid characters, so revert to default.
                        $length = strlen($this->tokens[$i]['content']);
                    }
                } else {
                    $length = strlen($this->tokens[$i]['content']);
                }

                $currColumn += $length;
            } else {
                $this->replaceTabsInToken($this->tokens[$i]);
                $length      = $this->tokens[$i]['length'];
                $currColumn += $length;
            }//end if

            $this->tokens[$i]['length'] = $length;

            if (isset($this->knownLengths[$this->tokens[$i]['code']]) === false
                && strpos($this->tokens[$i]['content'], $this->eolChar) !== false
            ) {
                $lineNumber++;
                $currColumn = 1;

                // Newline chars are not counted in the token length.
                $this->tokens[$i]['length'] -= $eolLen;
            }

            if ($checkAnnotations === true
                && ($this->tokens[$i]['code'] === T_COMMENT
                || $this->tokens[$i]['code'] === T_DOC_COMMENT_STRING
                || $this->tokens[$i]['code'] === T_DOC_COMMENT_TAG
                || ($inTests === true && $this->tokens[$i]['code'] === T_INLINE_HTML))
            ) {
                $commentText      = ltrim($this->tokens[$i]['content'], " \t/*");
                $commentText      = rtrim($commentText, " */\t\r\n");
                $commentTextLower = strtolower($commentText);
                if (strpos($commentText, '@codingStandards') !== false) {
                    // If this comment is the only thing on the line, it tells us
                    // to ignore the following line. If the line contains other content
                    // then we are just ignoring this one single line.
                    $ownLine = false;
                    if ($i > 0) {
                        for ($prev = ($i - 1); $prev >= 0; $prev--) {
                            if ($this->tokens[$prev]['code'] === T_WHITESPACE) {
                                continue;
                            }

                            break;
                        }

                        if ($this->tokens[$prev]['line'] !== $this->tokens[$i]['line']) {
                            $ownLine = true;
                        }
                    }

                    if ($ignoring === null
                        && strpos($commentText, '@codingStandardsIgnoreStart') !== false
                    ) {
                        $ignoring = ['all' => true];
                        if ($ownLine === true) {
                            $this->ignoredLines[$this->tokens[$i]['line']] = $ignoring;
                        }
                    } else if ($ignoring !== null
                        && strpos($commentText, '@codingStandardsIgnoreEnd') !== false
                    ) {
                        if ($ownLine === true) {
                            $this->ignoredLines[$this->tokens[$i]['line']] = ['all' => true];
                        } else {
                            $this->ignoredLines[$this->tokens[$i]['line']] = $ignoring;
                        }

                        $ignoring = null;
                    } else if ($ignoring === null
                        && strpos($commentText, '@codingStandardsIgnoreLine') !== false
                    ) {
                        $ignoring = ['all' => true];
                        if ($ownLine === true) {
                            $this->ignoredLines[$this->tokens[$i]['line']]       = $ignoring;
                            $this->ignoredLines[($this->tokens[$i]['line'] + 1)] = $ignoring;
                        } else {
                            $this->ignoredLines[$this->tokens[$i]['line']] = $ignoring;
                        }

                        $ignoring = null;
                    }//end if
                } else if (substr($commentTextLower, 0, 6) === 'phpcs:'
                    || substr($commentTextLower, 0, 7) === '@phpcs:'
                ) {
                    // If the @phpcs: syntax is being used, strip the @ to make
                    // comparisions easier.
                    if ($commentText[0] === '@') {
                        $commentText      = substr($commentText, 1);
                        $commentTextLower = strtolower($commentText);
                    }

                    // If there is a comment on the end, strip it off.
                    $commentStart = strpos($commentTextLower, ' --');
                    if ($commentStart !== false) {
                        $commentText      = substr($commentText, 0, $commentStart);
                        $commentTextLower = strtolower($commentText);
                    }

                    // If this comment is the only thing on the line, it tells us
                    // to ignore the following line. If the line contains other content
                    // then we are just ignoring this one single line.
                    $ownLine = false;
                    if ($i > 0) {
                        for ($prev = ($i - 1); $prev > 0; $prev--) {
                            if ($this->tokens[$prev]['code'] === T_WHITESPACE
                                || $this->tokens[$prev]['code'] === T_OPEN_TAG
                                || ($this->tokens[$prev]['code'] === T_INLINE_HTML
                                && trim($this->tokens[$prev]['content']) === '')
                            ) {
                                continue;
                            }

                            break;
                        }

                        if ($this->tokens[$prev]['line'] !== $this->tokens[$i]['line']) {
                            $ownLine = true;
                        }
                    }

                    if (substr($commentTextLower, 0, 9) === 'phpcs:set') {
                        // Ignore standards for complete lines that change sniff settings.
                        if ($ownLine === true) {
                            $this->ignoredLines[$this->tokens[$i]['line']] = true;
                        }

                        $this->tokens[$i]['code'] = T_PHPCS_SET;
                        $this->tokens[$i]['type'] = 'T_PHPCS_SET';
                    } else if (substr($commentTextLower, 0, 16) === 'phpcs:ignorefile') {
                        // The whole file will be ignored, but at least set the correct token.
                        $this->tokens[$i]['code'] = T_PHPCS_IGNORE_FILE;
                        $this->tokens[$i]['type'] = 'T_PHPCS_IGNORE_FILE';
                    } else if (substr($commentTextLower, 0, 13) === 'phpcs:disable') {
                        if ($ownLine === true) {
                            // Completely ignore the comment line.
                            $this->ignoredLines[$this->tokens[$i]['line']] = ['all' => true];
                        }

                        if ($ignoring === null) {
                            $ignoring = [];
                        }

                        $disabledSniffs = [];

                        $additionalText = substr($commentText, 14);
                        if ($additionalText === false) {
                            $ignoring = ['all' => true];
                        } else {
                            $parts = explode(',', substr($commentText, 13));
                            foreach ($parts as $sniffCode) {
                                $sniffCode = trim($sniffCode);
                                $disabledSniffs[$sniffCode] = true;
                                $ignoring[$sniffCode]       = true;
                            }
                        }

                        $this->tokens[$i]['code']       = T_PHPCS_DISABLE;
                        $this->tokens[$i]['type']       = 'T_PHPCS_DISABLE';
                        $this->tokens[$i]['sniffCodes'] = $disabledSniffs;
                    } else if (substr($commentTextLower, 0, 12) === 'phpcs:enable') {
                        if ($ignoring !== null) {
                            $enabledSniffs = [];

                            $additionalText = substr($commentText, 13);
                            if ($additionalText === false) {
                                $ignoring = null;
                            } else {
                                $parts = explode(',', substr($commentText, 13));
                                foreach ($parts as $sniffCode) {
                                    $sniffCode = trim($sniffCode);
                                    $enabledSniffs[$sniffCode] = true;
                                    if (isset($ignoring[$sniffCode]) === true) {
                                        unset($ignoring[$sniffCode]);
                                    }
                                }

                                if (empty($ignoring) === true) {
                                    $ignoring = null;
                                }
                            }

                            if ($ownLine === true) {
                                // Completely ignore the comment line.
                                $this->ignoredLines[$this->tokens[$i]['line']] = ['all' => true];
                            } else {
                                // The comment is on the same line as the code it is ignoring,
                                // so respect the new ignore rules.
                                $this->ignoredLines[$this->tokens[$i]['line']] = $ignoring;
                            }

                            $this->tokens[$i]['sniffCodes'] = $enabledSniffs;
                        }//end if

                        $this->tokens[$i]['code'] = T_PHPCS_ENABLE;
                        $this->tokens[$i]['type'] = 'T_PHPCS_ENABLE';
                    } else if (substr($commentTextLower, 0, 12) === 'phpcs:ignore') {
                        $ignoreRules = [];

                        $additionalText = substr($commentText, 13);
                        if ($additionalText === false) {
                            $ignoreRules = ['all' => true];
                        } else {
                            $parts = explode(',', substr($commentText, 13));
                            foreach ($parts as $sniffCode) {
                                $ignoreRules[trim($sniffCode)] = true;
                            }
                        }

                        $this->tokens[$i]['code']       = T_PHPCS_IGNORE;
                        $this->tokens[$i]['type']       = 'T_PHPCS_IGNORE';
                        $this->tokens[$i]['sniffCodes'] = $ignoreRules;

                        if ($ignoring !== null) {
                            $ignoreRules += $ignoring;
                        }

                        if ($ownLine === true) {
                            // Completely ignore the comment line, and set the folllowing
                            // line to include the ignore rules we've set.
                            $this->ignoredLines[$this->tokens[$i]['line']]       = ['all' => true];
                            $this->ignoredLines[($this->tokens[$i]['line'] + 1)] = $ignoreRules;
                        } else {
                            // The comment is on the same line as the code it is ignoring,
                            // so respect the ignore rules it set.
                            $this->ignoredLines[$this->tokens[$i]['line']] = $ignoreRules;
                        }
                    }//end if
                }//end if
            }//end if

            if ($ignoring !== null && isset($this->ignoredLines[$this->tokens[$i]['line']]) === false) {
                $this->ignoredLines[$this->tokens[$i]['line']] = $ignoring;
            }
        }//end for

    }//end createPositionMap()


    /**
     * Replaces tabs in original token content with spaces.
     *
     * Each tab can represent between 1 and $config->tabWidth spaces,
     * so this cannot be a straight string replace. The original content
     * is placed into an orig_content index and the new token length is also
     * set in the length index.
     *
     * @param array  $token    The token to replace tabs inside.
     * @param string $prefix   The character to use to represent the start of a tab.
     * @param string $padding  The character to use to represent the end of a tab.
     * @param int    $tabWidth The number of spaces each tab represents.
     *
     * @return void
     */
    public function replaceTabsInToken(&$token, $prefix=' ', $padding=' ', $tabWidth=null)
    {
        $checkEncoding = false;
        if (function_exists('iconv_strlen') === true) {
            $checkEncoding = true;
        }

        $currColumn = $token['column'];
        if ($tabWidth === null) {
            $tabWidth = $this->config->tabWidth;
            if ($tabWidth === 0) {
                $tabWidth = 1;
            }
        }

        if (rtrim($token['content'], "\t") === '') {
            // String only contains tabs, so we can shortcut the process.
            $numTabs = strlen($token['content']);

            $firstTabSize = ($tabWidth - (($currColumn - 1) % $tabWidth));
            $length       = ($firstTabSize + ($tabWidth * ($numTabs - 1)));
            $newContent   = $prefix.str_repeat($padding, ($length - 1));
        } else {
            // We need to determine the length of each tab.
            $tabs = explode("\t", $token['content']);

            $numTabs    = (count($tabs) - 1);
            $tabNum     = 0;
            $newContent = '';
            $length     = 0;

            foreach ($tabs as $content) {
                if ($content !== '') {
                    $newContent .= $content;
                    if ($checkEncoding === true) {
                        // Not using the default encoding, so take a bit more care.
                        $oldLevel = error_reporting();
                        error_reporting(0);
                        $contentLength = iconv_strlen($content, $this->config->encoding);
                        error_reporting($oldLevel);
                        if ($contentLength === false) {
                            // String contained invalid characters, so revert to default.
                            $contentLength = strlen($content);
                        }
{'fraction_non_alphanumeric': 0.12940619235179737, 'fraction_numerical': 0.004038490302637483, 'mean_word_length': 1.2145854035552612, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 267, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3600209', 'n_tokens_mistral': 10380, 'n_tokens_neox': 9659, 'n_words': 2434}