<commit_before><?php
/**
 * phpDocumentor
 *
 * PHP Version 5.3
 *
 * @copyright 2010-2012 <NAME> / Naenius (http://www.naenius.com)
 * @license   http://www.opensource.org/licenses/mit-license.php MIT
 * @link      http://phpdoc.org
 */

namespace phpDocumentor\Reflection;

/**
 * Reflection class for a full file.
 */
class FileReflector extends ReflectionAbstract implements \PHPParser_NodeVisitor
{
    protected $hash;
    protected $contents = 1;
    protected $includes = array();
    protected $constants = array();
    protected $classes = array();
    protected $traits = array();
    protected $interfaces = array();
    protected $functions = array();
    protected $filename = '';
    protected $doc_block;
    protected $default_package_name = 'Default';

    /** @var string[] A list of markers contained in this file. */
    protected $markers = array();

    /** @var string[] A list of errors during processing */
    protected $parse_markers = array();

    /** @var string[] A list of all marker types to search for in this file. */
    protected $marker_terms = array('TODO', 'FIXME');

    protected $namespace_aliases = array();
    protected $current_namespace = '';

    /**
     * Opens the file and retrieves its contents.
     *
     * During construction the given file is checked whether it is readable and
     * if the $validate argument is true a PHP Lint action is executed to
     * check whether the there are no parse errors.
     *
     * By default the Lint check is disable because of the performance hit
     * introduced by this action.
     *
     * If the validation checks out the file's contents are read; converted to
     * UTF-8 and the has is created from those contents.
     *
     * @param string  $file     Name of the file.
     * @param boolean $validate Whether to check the file using PHP Lint.
     *
     * @throws \phpDocumentor\Reflection\Exception when the filename is incorrect or
     *   the file can not be opened
     */
    public function __construct($file, $validate = false, $encoding = 'utf-8')
    {
        if (!is_string($file) || (!is_readable($file))) {
            throw new Exception\UnreadableFile(
                'The given file should be a string, should exist on the filesystem and should be readable'
            );
        }

        if ($validate) {
            exec('php -l ' . escapeshellarg($file), $output, $result);
            if ($result != 0) {
                throw new Exception\UnparsableFile(
                    'The given file could not be interpreted as it contains errors: ' . implode(PHP_EOL, $output)
                );
            }
        }

        $this->filename = $file;
        $this->contents = file_get_contents($file);

        if ($encoding !== 'utf-8') {
            $this->contents = iconv($encoding, 'utf-8//TRANSLIT', $this->contents);
        }

        // filemtime($file) is sometimes between 0.00001 and 0.00005 seconds faster but md5 is more accurate.
        // it can also result in false positives or false negatives after copying or checking out a codebase.
        $this->hash = md5($this->contents);
    }

    public function process()
    {
        // with big fluent interfaces it can happen that PHP-Parser's Traverser
        // exceeds the 100 recursions limit; we set it to 5000 to be sure.
        ini_set('xdebug.max_nesting_level', 5000);

        $traverser = new Traverser();
        $traverser->addVisitor($this);
        $traverser->traverse($this->contents);
<commit_msg>Due to feedback in the PR it was possible to make the conversion process more robust by adding //IGNORE to the conversion and by lowercasing the provided encoding to prevent comparison errors.<commit_after><?php
/**
 * phpDocumentor
 *
 * PHP Version 5.3
 *
 * @copyright 2010-2012 <NAME> / Naenius (http://www.naenius.com)
 * @license   http://www.opensource.org/licenses/mit-license.php MIT
 * @link      http://phpdoc.org
 */

namespace phpDocumentor\Reflection;

/**
 * Reflection class for a full file.
 */
class FileReflector extends ReflectionAbstract implements \PHPParser_NodeVisitor
{
    protected $hash;
    protected $contents = 1;
    protected $includes = array();
    protected $constants = array();
    protected $classes = array();
    protected $traits = array();
    protected $interfaces = array();
    protected $functions = array();
    protected $filename = '';
    protected $doc_block;
    protected $default_package_name = 'Default';

    /** @var string[] A list of markers contained in this file. */
    protected $markers = array();

    /** @var string[] A list of errors during processing */
    protected $parse_markers = array();

    /** @var string[] A list of all marker types to search for in this file. */
    protected $marker_terms = array('TODO', 'FIXME');

    protected $namespace_aliases = array();
    protected $current_namespace = '';

    /**
     * Opens the file and retrieves its contents.
     *
     * During construction the given file is checked whether it is readable and
     * if the $validate argument is true a PHP Lint action is executed to
     * check whether the there are no parse errors.
     *
     * By default the Lint check is disable because of the performance hit
     * introduced by this action.
     *
     * If the validation checks out the file's contents are read; converted to
     * UTF-8 and the has is created from those contents.
     *
     * @param string  $file     Name of the file.
     * @param boolean $validate Whether to check the file using PHP Lint.
     *
     * @throws \phpDocumentor\Reflection\Exception when the filename is incorrect or
     *   the file can not be opened
     */
    public function __construct($file, $validate = false, $encoding = 'utf-8')
    {
        if (!is_string($file) || (!is_readable($file))) {
            throw new Exception\UnreadableFile(
                'The given file should be a string, should exist on the filesystem and should be readable'
            );
        }

        if ($validate) {
            exec('php -l ' . escapeshellarg($file), $output, $result);
            if ($result != 0) {
                throw new Exception\UnparsableFile(
                    'The given file could not be interpreted as it contains errors: ' . implode(PHP_EOL, $output)
                );
            }
        }

        $this->filename = $file;
        $this->contents = file_get_contents($file);

        if (strtolower($encoding) !== 'utf-8') {
            $this->contents = iconv(strtolower($encoding), 'utf-8//IGNORE//TRANSLIT', $this->contents);
        }

        // filemtime($file) is sometimes between 0.00001 and 0.00005 seconds faster but md5 is more accurate.
        // it can also result in false positives or false negatives after copying or checking out a codebase.
        $this->hash = md5($this->contents);
    }

    public function process()
    {
        // with big fluent interfaces it can happen that PHP-Parser's Traverser
        // exceeds the 100 recursions limit; we set it to 5000 to be sure.
        ini_set('xdebug.max_nesting_level', 5000);

        $traverser = new Traverser();
        $traverser->addVisitor($this);
        $traverser->traverse($this->contents);

{'fraction_non_alphanumeric': 0.10522686525996414, 'fraction_numerical': 0.01130878499517308, 'mean_word_length': 2.5084663763909045, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5892238', 'n_tokens_mistral': 2043, 'n_tokens_neox': 1917, 'n_words': 725}