<commit_before><?php
/**
 * Tokens for PHP_Reflect parser
 *
 * @category PHP
 * @package  PHP_Reflect
 * @author   <NAME> <<EMAIL>>
 * @license  http://www.opensource.org/licenses/bsd-license.php  BSD
 * @version  Release: @package_version@
 * @link     http://php5.laurent-laville.org/reflect/
 */

abstract class PHP_Reflect_Token
{
    /**
     * @var string
     */
    protected $text;

    /**
     * @var integer
     */
    protected $line;

    /**
     * @var integer
     */
    protected $id;

    /**
     * @var array
     */
    protected $tokenStream;

    /**
     * Constructor.
     *
     * @param string  $text
     * @param integer $line
     * @param integer $id
     * @param array   $tokenStream
     */
    public function __construct($text, $line, $id, $tokens)
    {
        $this->text        = $text;
        $this->line        = $line;
        $this->id          = $id;
        $this->tokenStream = $tokens;
    }

    /**
     * @return string
     */
    public function __toString()
    {
        return $this->text;
    }

    /**
     * @return integer
     */
    public function getLine()
    {
        return $this->line;
    }
    
}

abstract class PHP_Reflect_TokenWithScope extends PHP_Reflect_Token
{
    protected $endTokenId;

    /**
     * Get the docblock for this token
     *
     * This method will fetch the docblock belonging to the current token. The
     * docblock must be placed on the line directly above the token to be
     * recognized.
     *
     * @return string|null Returns the docblock as a string if found
     */
    public function getDocblock()
    {
        $currentLineNumber = $this->tokenStream[$this->id][2];
        $prevLineNumber    = $currentLineNumber - 1;

        for ($i = $this->id - 1; $i > 0; $i--) {

            if ('T_FUNCTION' == $this->tokenStream[$i][0] 
                || 'T_CLASS' == $this->tokenStream[$i][0]
            ) {
                // Some other class or function, 
                // no docblock can be used for the current token
                break;
            }

            $line = $this->tokenStream[$i][2];

            if ($line == $currentLineNumber 
                || (($line == $prevLineNumber) && ('T_WHITESPACE' == $this->tokenStream[$i][0]))
            ) {
                break;
            }

            if (($line < $currentLineNumber) 
                && ('T_DOC_COMMENT' !== $this->tokenStream[$i][0])
                && ('T_COMMENT' !== $this->tokenStream[$i][0])
            ) {
                break;
            }

            return $this->tokenStream[$i][1];
        }
    }

    public function getVisibility()
    {
        for ($i = $this->id - 2; $i > $this->id - 7; $i -= 2) {
            if ($i < 0) {
                break;
            }

            if (true === ($this->tokenStream[$i][0] == 'T_PRIVATE' ||
                $this->tokenStream[$i][0] == 'T_PROTECTED' ||
                $this->tokenStream[$i][0] == 'T_PUBLIC')
            ) {
                return strtolower(
                    str_replace('T_', '', $this->tokenStream[$i][0])<commit_msg>getDocblock method accept more than just phpdoc comment style<commit_after><?php
/**
 * Tokens for PHP_Reflect parser
 *
 * @category PHP
 * @package  PHP_Reflect
 * @author   <NAME> <<EMAIL>>
 * @license  http://www.opensource.org/licenses/bsd-license.php  BSD
 * @version  Release: @package_version@
 * @link     http://php5.laurent-laville.org/reflect/
 */

abstract class PHP_Reflect_Token
{
    /**
     * @var string
     */
    protected $text;

    /**
     * @var integer
     */
    protected $line;

    /**
     * @var integer
     */
    protected $id;

    /**
     * @var array
     */
    protected $tokenStream;

    /**
     * Constructor.
     *
     * @param string  $text
     * @param integer $line
     * @param integer $id
     * @param array   $tokenStream
     */
    public function __construct($text, $line, $id, $tokens)
    {
        $this->text        = $text;
        $this->line        = $line;
        $this->id          = $id;
        $this->tokenStream = $tokens;
    }

    /**
     * @return string
     */
    public function __toString()
    {
        return $this->text;
    }

    /**
     * @return integer
     */
    public function getLine()
    {
        return $this->line;
    }
    
}

abstract class PHP_Reflect_TokenWithScope extends PHP_Reflect_Token
{
    protected $endTokenId;

    /**
     * Get the docblock for this token
     *
     * This method will fetch the docblock belonging to the current token. The
     * docblock must be placed on the line directly above the token to be
     * recognized.
     *
     * @return string|null Returns the docblock as a string if found
     */
    public function getDocblock()
    {
        $currentLineNumber = $this->tokenStream[$this->id][2];
        $prevLineNumber    = $currentLineNumber - 1;

        for ($i = $this->id - 1; $i > 0; $i--) {

            if ('T_FUNCTION' == $this->tokenStream[$i][0] 
                || 'T_CLASS' == $this->tokenStream[$i][0]
            ) {
                // Some other class or function, 
                // no docblock can be used for the current token
                break;
            }

            $line = $this->tokenStream[$i][2];

            if ($line == $currentLineNumber 
                || (($line == $prevLineNumber) && ('T_WHITESPACE' == $this->tokenStream[$i][0]))
            ) {
                continue;
            }

            if ($line == $prevLineNumber 
                && ('T_COMMENT' == $this->tokenStream[$i][0])
            ) {
                return $this->tokenStream[$i][1];
            }
            
            if (($line < $currentLineNumber) 
                && ('T_DOC_COMMENT' !== $this->tokenStream[$i][0])
            ) {
                break;
            }

            return $this->tokenStream[$i][1];
        }
    }

    public function getVisibility()
    {
        for ($i = $this->id - 2; $i > $this->id - 7; $i -= 2) {
            if ($i < 0) {
                break;
            }

            if (true === ($this->tokenStream[$i][0] == 'T_PRIVATE' ||
                $this->tokenStream[$i][0] == 'T_PROTECTED' ||
                $this->tokenStream[$i][0] == 'T_PUBLIC')
            ) {
                return strtolower(
                    str_replace('T_', '', $this->tokenStream[$i][0])
{'fraction_non_alphanumeric': 0.1467229782574691, 'fraction_numerical': 0.006413264508055686, 'mean_word_length': 1.5627254509018036, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6234327', 'n_tokens_mistral': 1989, 'n_tokens_neox': 1901, 'n_words': 436}