<commit_before><?php

namespace HyperDown;

/**
 * Parser 
 * 
 * @copyright Copyright (c) 2012 SegmentFault Team. (http://segmentfault.com)
 * @author Joyqi <<EMAIL>>
 * @license BSD License
 */
class Parser
{
    /**
     * _whiteList 
     * 
     * @var string
     */
    private $_commonWhiteList = 'kbd|b|i|strong|em|sup|sub|br|code|del|a|hr|small';

    /**
     * _specialWhiteList 
     * 
     * @var mixed
     * @access private
     */
    private $_specialWhiteList = [
        'table' =>  'table|tbody|thead|tfoot|tr|td|th'
    ];

    /**
     * _footnotes
     * 
     * @var array
     */
    private $_footnotes = [];

    /**
     * _blocks
     * 
     * @var array
     */
    private $_blocks = [];

    /**
     * _current  
     * 
     * @var string
     */
    private $_current = 'normal';

    /**
     * _pos  
     * 
     * @var int
     */
    private $_pos = -1;

    /**
     * _definitions  
     * 
     * @var array
     */
    private $_definitions = [];

    /**
     * @var array
     */
    private $_hooks = [];

    /**
     * @var array
     */
    private $_holders = [];

    /**
     * @var string
     */
    private $_uniqid = '';

    /**
     * @var int
     */
    private $_id = 0;

    /**
     * init uniqid
     */
    public function __construct()
    {
        $this->_uniqid = md5(uniqid());
    }

    /**
     * makeHtml  
     * 
     * @param mixed $text 
     * @return string
     */
    public function makeHtml($text)
    {
        $html = $this->parse($text);
        return $this->makeFootnotes($html);
    }

    /**
     * @param $type
     * @param $callback
     */
    public function hook($type, $callback)
    {
        $this->_hooks[$type][] = $callback;
    }

    /**
     * @param $html
     * @return string
     */
    private function makeFootnotes($html)
    {
        if (count($this->_footnotes) > 0) {
            $html .= '<div class="footnotes"><hr><ol>';
            $index = 1;

            while ($val = array_pop($this->_footnotes)) {
                if (is_string($val)) {
                    $val .= " <a href=\"#fnref-{$index}\" class=\"footnote-backref\">&#8617;</a>";
                } else {
                    $val[count($val) - 1] .= " <a href=\"#fnref-{$index}\" class=\"footnote-backref\">&#8617;</a>";
                    $val = count($val) > 1 ? $this->parse(implode("\n", $val)) : $this->parseInline($val[0]);
                }
<commit_msg>修正特殊的不可见字符引起的转义错误<commit_after><?php

namespace HyperDown;

/**
 * Parser 
 * 
 * @copyright Copyright (c) 2012 SegmentFault Team. (http://segmentfault.com)
 * @author Joyqi <<EMAIL>>
 * @license BSD License
 */
class Parser
{
    /**
     * _whiteList 
     * 
     * @var string
     */
    private $_commonWhiteList = 'kbd|b|i|strong|em|sup|sub|br|code|del|a|hr|small';

    /**
     * _specialWhiteList 
     * 
     * @var mixed
     * @access private
     */
    private $_specialWhiteList = [
        'table' =>  'table|tbody|thead|tfoot|tr|td|th'
    ];

    /**
     * _footnotes
     * 
     * @var array
     */
    private $_footnotes = [];

    /**
     * _blocks
     * 
     * @var array
     */
    private $_blocks = [];

    /**
     * _current  
     * 
     * @var string
     */
    private $_current = 'normal';

    /**
     * _pos  
     * 
     * @var int
     */
    private $_pos = -1;

    /**
     * _definitions  
     * 
     * @var array
     */
    private $_definitions = [];

    /**
     * @var array
     */
    private $_hooks = [];

    /**
     * @var array
     */
    private $_holders = [];

    /**
     * @var string
     */
    private $_uniqid = '';

    /**
     * @var int
     */
    private $_id = 0;

    /**
     * init uniqid
     */
    public function __construct()
    {
        $this->_uniqid = md5(uniqid());
    }

    /**
     * makeHtml  
     * 
     * @param mixed $text 
     * @return string
     */
    public function makeHtml($text)
    {
        $text = $this->initText($text);
        $html = $this->parse($text);
        return $this->makeFootnotes($html);
    }

    /**
     * @param $type
     * @param $callback
     */
    public function hook($type, $callback)
    {
        $this->_hooks[$type][] = $callback;
    }

    /**
     * @param $text
     * @return mixed
     */
    private function initText($text)
    {
        $text = str_replace(["\t", "\r"], ['    ', ''], $text);
        return $text;
    }

    /**
     * @param $html
     * @return string
     */
    private function makeFootnotes($html)
    {
        if (count($this->_footnotes) > 0) {
            $html .= '<div class="footnotes"><hr><ol>';
            $index = 1;

            while ($val = array_pop($this->_footnotes)) {
                if (is_string($val)) {
                    $val .= " <a href=\"#fnref-{$index}\" class=\"footnote-backref\">&#8617;</a>";
                } else {
                    $val[count($val) - 1] .= " <a href=\"#fnref-{$index}\" class=\"footnote-backref\">&#8617;</a>";
                    $val = count($val) > 1 ? $this->parse(implode("\n", $val)) : $this->parseInline($val[0]);
                }

{'fraction_non_alphanumeric': 0.18040132476134815, 'fraction_numerical': 0.007792713812585233, 'mean_word_length': 1.5303104977821587, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3887629', 'n_tokens_mistral': 1775, 'n_tokens_neox': 1694, 'n_words': 303}