<commit_before>package org.commonmark.internal;

import org.commonmark.parser.DelimiterProcessor;
import org.commonmark.internal.util.Parsing;
import org.commonmark.internal.util.Substring;
import org.commonmark.node.*;

import java.util.*;

public class DocumentParser {


    private static List<BlockParserFactory> CORE_FACTORIES = Arrays.<BlockParserFactory>asList(
            new IndentedCodeBlockParser.Factory(),
            new BlockQuoteParser.Factory(),
            new HeaderParser.Factory(),
            new FencedCodeBlockParser.Factory(),
            new HtmlBlockParser.Factory(),
            new HorizontalRuleParser.Factory(),
            new ListBlockParser.Factory());

    /**
     * 1-based line number
     */
    private int lineNumber = 0;
    private int lastLineLength = 0;

    private final InlineParser inlineParser;
    private final List<BlockParserFactory> blockParserFactories;

    private List<BlockParser> activeBlockParsers = new ArrayList<>();
    private Set<BlockParser> allBlockParsers = new HashSet<>();
    private Map<Node, Boolean> lastLineBlank = new HashMap<>();

    public DocumentParser(List<BlockParserFactory> customBlockParserFactories, List<DelimiterProcessor> delimiterProcessors) {
        blockParserFactories = new ArrayList<>(CORE_FACTORIES);
        blockParserFactories.addAll(customBlockParserFactories);
        inlineParser = new InlineParser(delimiterProcessors);
    }

    /**
     * The main parsing function. Returns a parsed document AST.
     */
    public Document parse(String input) {
        DocumentBlockParser documentBlockParser = new DocumentBlockParser();
        documentBlockParser.getBlock().setSourcePosition(new SourcePosition(1, 1));
        activateBlockParser(documentBlockParser);

        int lineStart = 0;
        int lineBreak;
        while ((lineBreak = Parsing.findLineBreak(input, lineStart)) != -1) {
            CharSequence line = Substring.of(input, lineStart, lineBreak);
            incorporateLine(line);
            if (lineBreak + 1 < input.length() && input.charAt(lineBreak) == '\r' && input.charAt(lineBreak + 1) == '\n') {
                lineStart = lineBreak + 2;
            } else {
                lineStart = lineBreak + 1;
            }
        }
        if (lineStart == 0 || lineStart < input.length()) {
            incorporateLine(Substring.of(input, lineStart, input.length()));
        }

        finalizeBlocks(activeBlockParsers, lineNumber);
        this.processInlines();
        return documentBlockParser.getBlock();
    }

    /**
     * Analyze a line of text and update the document appropriately. We parse markdown text by calling this on each
     * line of input, then finalizing the document.
     */
    private void incorporateLine(CharSequence ln) {
        int offset = 0;
        int nextNonSpace = 0;
        boolean blank = false;

        this.lineNumber += 1;

        ln = Parsing.prepareLine(ln);

        // For each containing block, try to parse the associated line start.
        // Bail out on failure: container will point to the last matching block.
        // Set all_matched to false if not all containers match.
        // The document will always match, can be skipped
        int matches = 1;
        for (BlockParser blockParser : activeBlockParsers.subList(1, activeBlockParsers.size())) {
            int match = Parsing.findNonSpace(ln, offset);
            if (match == -1) {
                nextNonSpace = ln.length();
                blank = true;
            } else {
                nextNonSpace = match;
                blank = false;
            }

            BlockParser.ContinueResult result = blockParser.continueBlock(ln, nextNonSpace, offset, blank);
            if (result instanceof BlockParser.BlockMatched) {
                BlockParser.BlockMatched blockMatched = (BlockParser.BlockMatched) result;
                offset = blockMatched.getNewOffset();
                matches++;
            } else if (result instanceof BlockParser.BlockMatchedAndCanBeFinalized) {
                finalize(blockParser, this.lineNumber);
                lastLineLength = ln.length() - 1; // -1 for newline
                return;
            } else if (result instanceof BlockParser.BlockDidNotMatch) {
                break;
            }
        }

        // TODO: Can we remove the closeUnmatchedBlocks calls?
        List<BlockParser> unmatchedBlockParsers = new ArrayList<>(activeBlockParsers.subList(matches, activeBlockParsers.size()));
        BlockParser blockParser = activeBlockParsers.get(matches - 1);
        boolean allClosed = unmatchedBlockParsers.isEmpty();

        // Check to see if we've hit 2nd blank line; if so break out of list:
        if (blank && isLastLineBlank(blockParser.getBlock())) {
            List<BlockParser> matchedBlockParsers = new ArrayList<>(activeBlockParsers.subList(0, matches));
            breakOutOfLists(matchedBlockParsers);
        }

        // Unless last matched container is a code block, try new container starts,
        // adding children to the last matched container:
        boolean blockStartsDone = false;
        while (!blockStartsDone) {
            int match = Parsing.findNonSpace(ln, offset);
            if (match == -1) {
                nextNonSpace = ln.length();
                blank = true;
                break;
            }
            nextNonSpace = match;
            blank = false;
            int indent = nextNonSpace - offset;

            if (!blockParser.shouldTryBlockStarts()) {
                break;
            }

            if (indent >= IndentedCodeBlockParser.INDENT && getActiveBlockParser().getBlock() instanceof Paragraph) {
                // An indented code block cannot interrupt a paragraph.
                offset = nextNonSpace;
                break;
            }

            // this is a little performance optimization:
            if (indent < IndentedCodeBlockParser.INDENT && Parsing.isLetter(ln, nextNonSpace)) {
                break;
            }

            blockStartsDone = true;
            for (BlockParserFactory blockParserFactory : blockParserFactories) {
                ParserStateImpl state = new ParserStateImpl(ln, offset, nextNonSpace, blockParser, lineNumber);
                BlockParserFactory.StartResult result = blockParserFactory.tryStart(state);
                if (result instanceof BlockParserFactory.BlockStart) {
                    BlockParserFactory.BlockStart blockStart = (BlockParserFactory.BlockStart) result;
                    allClosed = allClosed || finalizeBlocks(unmatchedBlockParsers);
                    offset = blockStart.getNewOffset();

                    if (blockStart.replaceActiveBlockParser()) {
                        removeActiveBlockParser();<commit_msg>Extract boolean for codeIndent<commit_after>package org.commonmark.internal;

import org.commonmark.parser.DelimiterProcessor;
import org.commonmark.internal.util.Parsing;
import org.commonmark.internal.util.Substring;
import org.commonmark.node.*;

import java.util.*;

public class DocumentParser {


    private static List<BlockParserFactory> CORE_FACTORIES = Arrays.<BlockParserFactory>asList(
            new IndentedCodeBlockParser.Factory(),
            new BlockQuoteParser.Factory(),
            new HeaderParser.Factory(),
            new FencedCodeBlockParser.Factory(),
            new HtmlBlockParser.Factory(),
            new HorizontalRuleParser.Factory(),
            new ListBlockParser.Factory());

    /**
     * 1-based line number
     */
    private int lineNumber = 0;
    private int lastLineLength = 0;

    private final InlineParser inlineParser;
    private final List<BlockParserFactory> blockParserFactories;

    private List<BlockParser> activeBlockParsers = new ArrayList<>();
    private Set<BlockParser> allBlockParsers = new HashSet<>();
    private Map<Node, Boolean> lastLineBlank = new HashMap<>();

    public DocumentParser(List<BlockParserFactory> customBlockParserFactories, List<DelimiterProcessor> delimiterProcessors) {
        blockParserFactories = new ArrayList<>(CORE_FACTORIES);
        blockParserFactories.addAll(customBlockParserFactories);
        inlineParser = new InlineParser(delimiterProcessors);
    }

    /**
     * The main parsing function. Returns a parsed document AST.
     */
    public Document parse(String input) {
        DocumentBlockParser documentBlockParser = new DocumentBlockParser();
        documentBlockParser.getBlock().setSourcePosition(new SourcePosition(1, 1));
        activateBlockParser(documentBlockParser);

        int lineStart = 0;
        int lineBreak;
        while ((lineBreak = Parsing.findLineBreak(input, lineStart)) != -1) {
            CharSequence line = Substring.of(input, lineStart, lineBreak);
            incorporateLine(line);
            if (lineBreak + 1 < input.length() && input.charAt(lineBreak) == '\r' && input.charAt(lineBreak + 1) == '\n') {
                lineStart = lineBreak + 2;
            } else {
                lineStart = lineBreak + 1;
            }
        }
        if (lineStart == 0 || lineStart < input.length()) {
            incorporateLine(Substring.of(input, lineStart, input.length()));
        }

        finalizeBlocks(activeBlockParsers, lineNumber);
        this.processInlines();
        return documentBlockParser.getBlock();
    }

    /**
     * Analyze a line of text and update the document appropriately. We parse markdown text by calling this on each
     * line of input, then finalizing the document.
     */
    private void incorporateLine(CharSequence ln) {
        int offset = 0;
        int nextNonSpace = 0;
        boolean blank = false;

        this.lineNumber += 1;

        ln = Parsing.prepareLine(ln);

        // For each containing block, try to parse the associated line start.
        // Bail out on failure: container will point to the last matching block.
        // Set all_matched to false if not all containers match.
        // The document will always match, can be skipped
        int matches = 1;
        for (BlockParser blockParser : activeBlockParsers.subList(1, activeBlockParsers.size())) {
            int match = Parsing.findNonSpace(ln, offset);
            if (match == -1) {
                nextNonSpace = ln.length();
                blank = true;
            } else {
                nextNonSpace = match;
                blank = false;
            }

            BlockParser.ContinueResult result = blockParser.continueBlock(ln, nextNonSpace, offset, blank);
            if (result instanceof BlockParser.BlockMatched) {
                BlockParser.BlockMatched blockMatched = (BlockParser.BlockMatched) result;
                offset = blockMatched.getNewOffset();
                matches++;
            } else if (result instanceof BlockParser.BlockMatchedAndCanBeFinalized) {
                finalize(blockParser, this.lineNumber);
                lastLineLength = ln.length() - 1; // -1 for newline
                return;
            } else if (result instanceof BlockParser.BlockDidNotMatch) {
                break;
            }
        }

        // TODO: Can we remove the closeUnmatchedBlocks calls?
        List<BlockParser> unmatchedBlockParsers = new ArrayList<>(activeBlockParsers.subList(matches, activeBlockParsers.size()));
        BlockParser blockParser = activeBlockParsers.get(matches - 1);
        boolean allClosed = unmatchedBlockParsers.isEmpty();

        // Check to see if we've hit 2nd blank line; if so break out of list:
        if (blank && isLastLineBlank(blockParser.getBlock())) {
            List<BlockParser> matchedBlockParsers = new ArrayList<>(activeBlockParsers.subList(0, matches));
            breakOutOfLists(matchedBlockParsers);
        }

        // Unless last matched container is a code block, try new container starts,
        // adding children to the last matched container:
        boolean blockStartsDone = false;
        while (!blockStartsDone) {
            int match = Parsing.findNonSpace(ln, offset);
            if (match == -1) {
                nextNonSpace = ln.length();
                blank = true;
                break;
            }
            nextNonSpace = match;
            blank = false;
            int indent = nextNonSpace - offset;
            boolean codeIndent = indent >= IndentedCodeBlockParser.INDENT;

            if (!blockParser.shouldTryBlockStarts()) {
                break;
            }

            if (codeIndent && getActiveBlockParser().getBlock() instanceof Paragraph) {
                // An indented code block cannot interrupt a paragraph.
                offset = nextNonSpace;
                break;
            }

            // this is a little performance optimization:
            if (!codeIndent && Parsing.isLetter(ln, nextNonSpace)) {
                break;
            }

            blockStartsDone = true;
            for (BlockParserFactory blockParserFactory : blockParserFactories) {
                ParserStateImpl state = new ParserStateImpl(ln, offset, nextNonSpace, blockParser, lineNumber);
                BlockParserFactory.StartResult result = blockParserFactory.tryStart(state);
                if (result instanceof BlockParserFactory.BlockStart) {
                    BlockParserFactory.BlockStart blockStart = (BlockParserFactory.BlockStart) result;
                    allClosed = allClosed || finalizeBlocks(unmatchedBlockParsers);
                    offset = blockStart.getNewOffset();

                    if (blockStart.replaceActiveBlockParser()) {
                        removeActiveBlockParser();
{'fraction_non_alphanumeric': 0.08676837318468535, 'fraction_numerical': 0.003520610239108112, 'mean_word_length': 2.2627422828427854, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2370694', 'n_tokens_mistral': 3450, 'n_tokens_neox': 3292, 'n_words': 892}