<commit_before><?php

/*
 * This file is part of Psy Shell.
 *
 * (c) 2012-2020 <NAME>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Psy\Formatter;

use Psy\Exception\RuntimeException;
use Symfony\Component\Console\Formatter\OutputFormatter;

/**
 * A pretty-printer for code.
 */
class CodeFormatter implements Formatter
{
    const LINE_MARKER    = '  <urgent>></urgent> ';
    const NO_LINE_MARKER = '    ';

    const HIGHLIGHT_DEFAULT     = 'default';
    const HIGHLIGHT_KEYWORD     = 'keyword';

    const HIGHLIGHT_PUBLIC      = 'public';
    const HIGHLIGHT_PROTECTED   = 'protected';
    const HIGHLIGHT_PRIVATE     = 'private';

    const HIGHLIGHT_CONST       = 'const';
    const HIGHLIGHT_NUMBER      = 'number';
    const HIGHLIGHT_STRING      = 'string';
    const HIGHLIGHT_COMMENT     = 'comment';
    const HIGHLIGHT_INLINE_HTML = 'inline_html';

    private static $tokenMap = [
        // Not highlighted
        \T_OPEN_TAG           => self::HIGHLIGHT_DEFAULT,
        \T_OPEN_TAG_WITH_ECHO => self::HIGHLIGHT_DEFAULT,
        \T_CLOSE_TAG          => self::HIGHLIGHT_DEFAULT,
        \T_STRING             => self::HIGHLIGHT_DEFAULT,
        \T_VARIABLE           => self::HIGHLIGHT_DEFAULT,
        \T_NS_SEPARATOR       => self::HIGHLIGHT_DEFAULT,

        // Visibility
        \T_PUBLIC    => self::HIGHLIGHT_PUBLIC,
        \T_PROTECTED => self::HIGHLIGHT_PROTECTED,
        \T_PRIVATE   => self::HIGHLIGHT_PRIVATE,

        // Constants
        \T_DIR      => self::HIGHLIGHT_CONST,
        \T_FILE     => self::HIGHLIGHT_CONST,
        \T_METHOD_C => self::HIGHLIGHT_CONST,
        \T_NS_C     => self::HIGHLIGHT_CONST,
        \T_LINE     => self::HIGHLIGHT_CONST,
        \T_CLASS_C  => self::HIGHLIGHT_CONST,
        \T_FUNC_C   => self::HIGHLIGHT_CONST,
        \T_TRAIT_C  => self::HIGHLIGHT_CONST,

        // Types
        \T_DNUMBER                  => self::HIGHLIGHT_NUMBER,
        \T_LNUMBER                  => self::HIGHLIGHT_NUMBER,
        \T_ENCAPSED_AND_WHITESPACE  => self::HIGHLIGHT_STRING,
        \T_CONSTANT_ENCAPSED_STRING => self::HIGHLIGHT_STRING,

        // Comments
        \T_COMMENT     => self::HIGHLIGHT_COMMENT,
        \T_DOC_COMMENT => self::HIGHLIGHT_COMMENT,

        // @todo something better here?
        \T_INLINE_HTML => self::HIGHLIGHT_INLINE_HTML,
    ];

    /**
     * Format the code represented by $reflector for shell output.
     *
     * @param \Reflector  $reflector
     * @param string|null $colorMode (deprecated and ignored)
     *
     * @return string formatted code
     */
    public static function format(\Reflector $reflector, $colorMode = null)
    {
        if (self::isReflectable($reflector)) {
            if ($code = @\file_get_contents($reflector->getFileName())) {
                return self::formatCode($code, $reflector->getStartLine(), $reflector->getEndLine());
            }
        }

        throw new RuntimeException('Source code unavailable');
    }

    /**
     * Format code for shell output.
     *
     * Optionally, restrict by $startLine and $endLine line numbers, or pass $markLine to add a line marker.
     *
     * @param string      $code
     * @param number      $startLine
     * @param number|null $endLine
     * @param number|null $markLine
     *
     * @return string formatted code
     */
    public static function formatCode($code, $startLine = 1, $endLine = null, $markLine = null)
    {
        $spans = self::tokenizeSpans($code);
        $lines = self::splitLines($spans, $startLine, $endLine);
        $lines = self::formatLines($lines);
        $lines = self::numberLines($lines, $markLine);

        return \implode('', \iterator_to_array($lines));
    }

    /**
     * Split code into highlight spans.
     *
     * Tokenize via \token_get_all, then map these tokens to internal highlight types, combining
     * adjacent spans of the same highlight type.
     *
     * @todo consider switching \token_get_all() out for PHP-Parser-based formatting at some point.
     *
     * @param string $code
     *
     * @return \Generator [$spanType, $spanText] highlight spans
     */
    private static function tokenizeSpans($code)
    {
        $spanType = null;
        $nextType = null;
        $buffer   = '';

        foreach (\token_get_all($code) as $token) {
            $nextType = self::nextHighlightType($token, $spanType);<commit_msg>Include doc comments in `show` command output for classes, functions and methods.<commit_after><?php

/*
 * This file is part of Psy Shell.
 *
 * (c) 2012-2020 <NAME>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Psy\Formatter;

use Psy\Exception\RuntimeException;
use Symfony\Component\Console\Formatter\OutputFormatter;

/**
 * A pretty-printer for code.
 */
class CodeFormatter implements Formatter
{
    const LINE_MARKER    = '  <urgent>></urgent> ';
    const NO_LINE_MARKER = '    ';

    const HIGHLIGHT_DEFAULT     = 'default';
    const HIGHLIGHT_KEYWORD     = 'keyword';

    const HIGHLIGHT_PUBLIC      = 'public';
    const HIGHLIGHT_PROTECTED   = 'protected';
    const HIGHLIGHT_PRIVATE     = 'private';

    const HIGHLIGHT_CONST       = 'const';
    const HIGHLIGHT_NUMBER      = 'number';
    const HIGHLIGHT_STRING      = 'string';
    const HIGHLIGHT_COMMENT     = 'comment';
    const HIGHLIGHT_INLINE_HTML = 'inline_html';

    private static $tokenMap = [
        // Not highlighted
        \T_OPEN_TAG           => self::HIGHLIGHT_DEFAULT,
        \T_OPEN_TAG_WITH_ECHO => self::HIGHLIGHT_DEFAULT,
        \T_CLOSE_TAG          => self::HIGHLIGHT_DEFAULT,
        \T_STRING             => self::HIGHLIGHT_DEFAULT,
        \T_VARIABLE           => self::HIGHLIGHT_DEFAULT,
        \T_NS_SEPARATOR       => self::HIGHLIGHT_DEFAULT,

        // Visibility
        \T_PUBLIC    => self::HIGHLIGHT_PUBLIC,
        \T_PROTECTED => self::HIGHLIGHT_PROTECTED,
        \T_PRIVATE   => self::HIGHLIGHT_PRIVATE,

        // Constants
        \T_DIR      => self::HIGHLIGHT_CONST,
        \T_FILE     => self::HIGHLIGHT_CONST,
        \T_METHOD_C => self::HIGHLIGHT_CONST,
        \T_NS_C     => self::HIGHLIGHT_CONST,
        \T_LINE     => self::HIGHLIGHT_CONST,
        \T_CLASS_C  => self::HIGHLIGHT_CONST,
        \T_FUNC_C   => self::HIGHLIGHT_CONST,
        \T_TRAIT_C  => self::HIGHLIGHT_CONST,

        // Types
        \T_DNUMBER                  => self::HIGHLIGHT_NUMBER,
        \T_LNUMBER                  => self::HIGHLIGHT_NUMBER,
        \T_ENCAPSED_AND_WHITESPACE  => self::HIGHLIGHT_STRING,
        \T_CONSTANT_ENCAPSED_STRING => self::HIGHLIGHT_STRING,

        // Comments
        \T_COMMENT     => self::HIGHLIGHT_COMMENT,
        \T_DOC_COMMENT => self::HIGHLIGHT_COMMENT,

        // @todo something better here?
        \T_INLINE_HTML => self::HIGHLIGHT_INLINE_HTML,
    ];

    /**
     * Format the code represented by $reflector for shell output.
     *
     * @param \Reflector  $reflector
     * @param string|null $colorMode (deprecated and ignored)
     *
     * @return string formatted code
     */
    public static function format(\Reflector $reflector, $colorMode = null)
    {
        if (self::isReflectable($reflector)) {
            if ($code = @\file_get_contents($reflector->getFileName())) {
                return self::formatCode($code, self::getStartLine($reflector), $reflector->getEndLine());
            }
        }

        throw new RuntimeException('Source code unavailable');
    }

    /**
     * Format code for shell output.
     *
     * Optionally, restrict by $startLine and $endLine line numbers, or pass $markLine to add a line marker.
     *
     * @param string      $code
     * @param number      $startLine
     * @param number|null $endLine
     * @param number|null $markLine
     *
     * @return string formatted code
     */
    public static function formatCode($code, $startLine = 1, $endLine = null, $markLine = null)
    {
        $spans = self::tokenizeSpans($code);
        $lines = self::splitLines($spans, $startLine, $endLine);
        $lines = self::formatLines($lines);
        $lines = self::numberLines($lines, $markLine);

        return \implode('', \iterator_to_array($lines));
    }

    /**
     * Get the start line for a given Reflector.
     *
     * Tries to incorporate doc comments if possible.
     *
     * This is typehinted as \Reflector but we've narrowed the input via self::isReflectable already.
     *
     * @param \ReflectionClass|\ReflectionFunctionAbstract $reflector
     *
     * @return number
     */
    private static function getStartLine(\Reflector $reflector)
    {
        $startLine = $reflector->getStartLine();

        if ($docComment = $reflector->getDocComment()) {
            $startLine -= \preg_match_all('/(\r\n?|\n)/', $docComment) + 1;
        }

        return \max($startLine, 1);
    }

    /**
     * Split code into highlight spans.
     *
     * Tokenize via \token_get_all, then map these tokens to internal highlight types, combining
     * adjacent spans of the same highlight type.
     *
     * @todo consider switching \token_get_all() out for PHP-Parser-based formatting at some point.
     *
     * @param string $code
     *
     * @return \Generator [$spanType, $spanText] highlight spans
     */
    private static function tokenizeSpans($code)
    {
        $spanType = null;
        $nextType = null;
        $buffer   = '';

        foreach (\token_get_all($code) as $token) {
            $nextType = self::nextHighlightType($token, $spanType);
{'fraction_non_alphanumeric': 0.11215435410647957, 'fraction_numerical': 0.0020635575732562937, 'mean_word_length': 2.3714782608695653, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7225809', 'n_tokens_mistral': 3077, 'n_tokens_neox': 2988, 'n_words': 668}