<commit_before><?php

namespace FluxBB\CommonMark;

use FluxBB\CommonMark\Common\Collection;
use FluxBB\CommonMark\Common\Text;
use FluxBB\CommonMark\Node\Container;
use FluxBB\CommonMark\Node\Document;
use FluxBB\CommonMark\Parser\AbstractBlockParser;
use FluxBB\CommonMark\Parser\Block\AtxHeaderParser;
use FluxBB\CommonMark\Parser\Block\BlockquoteParser;
use FluxBB\CommonMark\Parser\Block\CodeBlockParser;
use FluxBB\CommonMark\Parser\Block\FencedCodeBlockParser;
use FluxBB\CommonMark\Parser\Block\HorizontalRuleParser;
use FluxBB\CommonMark\Parser\Block\HTMLBlockParser;
use FluxBB\CommonMark\Parser\Block\LinkReferenceParser;
use FluxBB\CommonMark\Parser\Block\ListParser;
use FluxBB\CommonMark\Parser\Block\ParagraphParser;
use FluxBB\CommonMark\Parser\Block\SetextHeaderParser;
use FluxBB\CommonMark\Parser\BlockParserInterface;

class DocumentParser implements BlockParserInterface
{

    /**
     * @var Collection
     */
    protected $links;

    /**
     * @var Collection
     */
    protected $titles;

    /**
     * @var BlockParserInterface[]
     */
    protected $parsers = [];


    /**
     * Create a parser instance.
     */
    public function __construct()
    {
        $this->links = new Collection();
        $this->titles = new Collection();

        $this->registerDefaultParsers();
    }

    /**
     * Parse the given Markdown text into a document tree.
     *
     * @param string $markdown
     * @return Document
     */
    public function convert($markdown)
    {
        $root = new Document();

        $parser = $this->buildParserStack();

        $text = new Text($markdown);
        $this->prepare($text);

        $parser->parseBlock($text, $root);

        $this->parseInlineContent($root);

        return $root;
    }

    /**
     * Preprocess the text.
     *
     * This unifies line endings and replaces tabs by spaces.
     *
     * @param Text $text
     * @return void
     */
    protected function prepare(Text $text)
    {
        // Unify line endings
        $text->replaceString("\r\n", "\n");
        $text->replaceString("\r", "\n");

        // Replace tabs by spaces
        $text->replace('/(.*?)\t/', function (Text $whole, Text $string) {
            $tabWidth = 4;
            return $string . str_repeat(' ', $tabWidth - $string->getLength() % $tabWidth);
        });
    }

    /**
     * Parse the inline elements of our document tree.
     *
     * @param Document $root
     * @return void
     */
    protected function parseInlineContent(Document $root)
    {
        $inlineParser = new InlineParser($this->links, $this->titles);
        $root->visit($inlineParser);
    }

    /**
     * Register all standard parsers.
     *
     * @return void
     */
    protected function registerDefaultParsers()
    {
        $this->parsers = [
            new ListParser(),
            new HTMLBlockParser(),
            new FencedCodeBlockParser(),
            new CodeBlockParser(),
            new BlockquoteParser(),
            new LinkReferenceParser($this->links, $this->titles),
            new SetextHeaderParser(),
            new HorizontalRuleParser(),
            new AtxHeaderParser(),
            new ParagraphParser(),
        ];
    }

    /**
     * Build the nested stack of closures that executes the parsers in the correct order.
     *
     * @return BlockParserInterface
     */
    protected function buildParserStack()<commit_msg>Try to fix precedence of list parser vs. horizontal rules.<commit_after><?php

namespace FluxBB\CommonMark;

use FluxBB\CommonMark\Common\Collection;
use FluxBB\CommonMark\Common\Text;
use FluxBB\CommonMark\Node\Container;
use FluxBB\CommonMark\Node\Document;
use FluxBB\CommonMark\Parser\AbstractBlockParser;
use FluxBB\CommonMark\Parser\Block\AtxHeaderParser;
use FluxBB\CommonMark\Parser\Block\BlockquoteParser;
use FluxBB\CommonMark\Parser\Block\CodeBlockParser;
use FluxBB\CommonMark\Parser\Block\FencedCodeBlockParser;
use FluxBB\CommonMark\Parser\Block\HorizontalRuleParser;
use FluxBB\CommonMark\Parser\Block\HTMLBlockParser;
use FluxBB\CommonMark\Parser\Block\LinkReferenceParser;
use FluxBB\CommonMark\Parser\Block\ListParser;
use FluxBB\CommonMark\Parser\Block\ParagraphParser;
use FluxBB\CommonMark\Parser\Block\SetextHeaderParser;
use FluxBB\CommonMark\Parser\BlockParserInterface;

class DocumentParser implements BlockParserInterface
{

    /**
     * @var Collection
     */
    protected $links;

    /**
     * @var Collection
     */
    protected $titles;

    /**
     * @var BlockParserInterface[]
     */
    protected $parsers = [];


    /**
     * Create a parser instance.
     */
    public function __construct()
    {
        $this->links = new Collection();
        $this->titles = new Collection();

        $this->registerDefaultParsers();
    }

    /**
     * Parse the given Markdown text into a document tree.
     *
     * @param string $markdown
     * @return Document
     */
    public function convert($markdown)
    {
        $root = new Document();

        $parser = $this->buildParserStack();

        $text = new Text($markdown);
        $this->prepare($text);

        $parser->parseBlock($text, $root);

        $this->parseInlineContent($root);

        return $root;
    }

    /**
     * Preprocess the text.
     *
     * This unifies line endings and replaces tabs by spaces.
     *
     * @param Text $text
     * @return void
     */
    protected function prepare(Text $text)
    {
        // Unify line endings
        $text->replaceString("\r\n", "\n");
        $text->replaceString("\r", "\n");

        // Replace tabs by spaces
        $text->replace('/(.*?)\t/', function (Text $whole, Text $string) {
            $tabWidth = 4;
            return $string . str_repeat(' ', $tabWidth - $string->getLength() % $tabWidth);
        });
    }

    /**
     * Parse the inline elements of our document tree.
     *
     * @param Document $root
     * @return void
     */
    protected function parseInlineContent(Document $root)
    {
        $inlineParser = new InlineParser($this->links, $this->titles);
        $root->visit($inlineParser);
    }

    /**
     * Register all standard parsers.
     *
     * @return void
     */
    protected function registerDefaultParsers()
    {
        $this->parsers = [
            new HTMLBlockParser(),
            new FencedCodeBlockParser(),
            new CodeBlockParser(),
            new HorizontalRuleParser(),
            new ListParser(),
            new BlockquoteParser(),
            new LinkReferenceParser($this->links, $this->titles),
            new SetextHeaderParser(),
            new AtxHeaderParser(),
            new ParagraphParser(),
        ];
    }

    /**
     * Build the nested stack of closures that executes the parsers in the correct order.
     *
     * @return BlockParserInterface
     */
    protected function buildParserStack()
{'fraction_non_alphanumeric': 0.11918225315354503, 'fraction_numerical': 0.0002899811512251704, 'mean_word_length': 2.4182358771060457, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '318355', 'n_tokens_mistral': 1986, 'n_tokens_neox': 1924, 'n_words': 436}