<commit_before>package eu.mrico.creole.impl;

import eu.mrico.creole.CreoleException;
import eu.mrico.creole.CreoleParser;
import eu.mrico.creole.ast.Bold;
import eu.mrico.creole.ast.Cell;
import eu.mrico.creole.ast.Document;
import eu.mrico.creole.ast.Element;
import eu.mrico.creole.ast.Heading;
import eu.mrico.creole.ast.HorizontalRule;
import eu.mrico.creole.ast.Image;
import eu.mrico.creole.ast.Italic;
import eu.mrico.creole.ast.LineBreak;
import eu.mrico.creole.ast.Link;
import eu.mrico.creole.ast.List;
import eu.mrico.creole.ast.ListItem;
import eu.mrico.creole.ast.Paragraph;
import eu.mrico.creole.ast.Preformatted;
import eu.mrico.creole.ast.Row;
import eu.mrico.creole.ast.Table;
import eu.mrico.creole.ast.Text;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;

/**
 *
 * @author <NAME>
 */
class CreoleParserImpl implements CreoleParser {

    private BufferedReader reader;
    private String currentLine;
    private Element parent;

    @Override
    public Document parse(InputStream in) throws CreoleException {
        return parse(new InputStreamReader(in));
    }

    @Override
    public Document parse(Reader stream) throws CreoleException {
        this.reader = new BufferedReader(stream);

        Document document = new Document();
        this.parent = document;
        String line = null;

        boolean newParagraph = true;

        try {
            while ((line = reader.readLine()) != null) {
                line = line.trim();
                if (line.length() == 0) {
                    // new block
                    this.parent = new Paragraph();
                    document.add(parent);
                    newParagraph = true;

                } else if (line.startsWith("=")) {
                    document.add(parseHeading(line));

                } else if (line.startsWith("* ")) {
                    // unordered list
                    parent.add(parseList(line, false));

                } else if (line.startsWith("# ")) {
                    // ordered list
                    parent.add(parseList(line, true));

                } else if (line.startsWith("|")) {
                    // table
                    parent.add(parseTable(line));

                } else if (line.equals("{{{")) {
                    parent.add(readPreformatted());

                } else if (line.matches("\\{\\{[^\\{].*")) {
                    // TODO: find better way to recognize image pattern
                    // image
                    parent.add(readImage(line));

                } else if (line.startsWith("----")) {
                    document.add(new HorizontalRule());

                } else {
                    if (!newParagraph) {
                        parent.add(new Text(" "));
                    }
<commit_msg>fixed heading -> paragraph handling<commit_after>package eu.mrico.creole.impl;

import eu.mrico.creole.CreoleException;
import eu.mrico.creole.CreoleParser;
import eu.mrico.creole.ast.Bold;
import eu.mrico.creole.ast.Cell;
import eu.mrico.creole.ast.Document;
import eu.mrico.creole.ast.Element;
import eu.mrico.creole.ast.Heading;
import eu.mrico.creole.ast.HorizontalRule;
import eu.mrico.creole.ast.Image;
import eu.mrico.creole.ast.Italic;
import eu.mrico.creole.ast.LineBreak;
import eu.mrico.creole.ast.Link;
import eu.mrico.creole.ast.List;
import eu.mrico.creole.ast.ListItem;
import eu.mrico.creole.ast.Paragraph;
import eu.mrico.creole.ast.Preformatted;
import eu.mrico.creole.ast.Row;
import eu.mrico.creole.ast.Table;
import eu.mrico.creole.ast.Text;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;

/**
 *
 * @author <NAME>
 */
class CreoleParserImpl implements CreoleParser {

    private BufferedReader reader;
    private String currentLine;
    private Element parent;

    @Override
    public Document parse(InputStream in) throws CreoleException {
        return parse(new InputStreamReader(in));
    }

    @Override
    public Document parse(Reader stream) throws CreoleException {
        this.reader = new BufferedReader(stream);

        Document document = new Document();
        this.parent = document;
        String line = null;

        boolean newParagraph = true;

        try {
            while ((line = reader.readLine()) != null) {
                line = line.trim();
                if (line.length() == 0) {
                    // new block
                    this.parent = new Paragraph();
                    document.add(parent);
                    newParagraph = true;

                } else if (line.startsWith("=")) {
                    document.add(parseHeading(line));

                    this.parent = new Paragraph();
                    document.add(parent);
                    newParagraph = true;

                } else if (line.startsWith("* ")) {
                    // unordered list
                    parent.add(parseList(line, false));

                } else if (line.startsWith("# ")) {
                    // ordered list
                    parent.add(parseList(line, true));

                } else if (line.startsWith("|")) {
                    // table
                    parent.add(parseTable(line));

                } else if (line.equals("{{{")) {
                    parent.add(readPreformatted());

                } else if (line.matches("\\{\\{[^\\{].*")) {
                    // TODO: find better way to recognize image pattern
                    // image
                    parent.add(readImage(line));

                } else if (line.startsWith("----")) {
                    document.add(new HorizontalRule());

                } else {
                    if (!newParagraph) {
                        parent.add(new Text(" "));
                    }

{'fraction_non_alphanumeric': 0.12196754224527355, 'fraction_numerical': 0.00033461602810774636, 'mean_word_length': 1.8372093023255813, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3831192', 'n_tokens_mistral': 1743, 'n_tokens_neox': 1652, 'n_words': 343}