<commit_before>// Copyright (c) 2014 <NAME>
// Licensed under MIT license.

package md

import (
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
)

func ParseFile(path string, c MarkdownConverter) (md Markdown) {
	file, err := os.Open(path)
	if err != nil {
		fmt.Println("Error opening file", err)
		return
	}
	defer file.Close()
	b, _ := ioutil.ReadAll(file)
	return Parse(string(b), c)
}

func Parse(lines string, c MarkdownConverter) (md Markdown) {
	buf := []Inline{}
	for _, s := range strings.Split(lines, "\n") {
		if strings.HasPrefix(s, "# ") {
			// H1
			md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToH1, Values: parseInline(trimHeaderChars(s))})
		} else if strings.HasPrefix(s, "## ") {
			// H2
			md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToH2, Values: parseInline(trimHeaderChars(s))})
		} else if strings.HasPrefix(s, "### ") {
			// H3
			md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToH3, Values: parseInline(trimHeaderChars(s))})
		} else if strings.HasPrefix(s, "#### ") {
			// H4
			md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToH4, Values: parseInline(trimHeaderChars(s))})
		} else if strings.HasPrefix(s, "##### ") {
			// H5
			md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToH5, Values: parseInline(trimHeaderChars(s))})
		} else if strings.HasPrefix(s, "###### ") {
			// H6
			md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToH6, Values: parseInline(trimHeaderChars(s))})
		} else if s == "" {
			if 0 < len(buf) {
				// End of paragraph
				md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToP, Values: buf})
				buf = []Inline{}
			}
		} else if 0 < len(buf) && strings.Replace(s, "=", "", -1) == "" {
			// Last line is H1
			if 1 < len(buf) {
				md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToP, Values: buf[:len(buf)-1]})
			}
			md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToH1, Values: buf[len(buf)-1:]})
			buf = []Inline{}
		} else if 0 < len(buf) && strings.Replace(s, "-", "", -1) == "" {
			// Last line is H2<commit_msg>Reversed order of header evaluation and ignored space after header marks.<commit_after>// Copyright (c) 2014 <NAME>
// Licensed under MIT license.

package md

import (
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
)

func ParseFile(path string, c MarkdownConverter) (md Markdown) {
	file, err := os.Open(path)
	if err != nil {
		fmt.Println("Error opening file", err)
		return
	}
	defer file.Close()
	b, _ := ioutil.ReadAll(file)
	return Parse(string(b), c)
}

func Parse(lines string, c MarkdownConverter) (md Markdown) {
	buf := []Inline{}
	for _, s := range strings.Split(lines, "\n") {
		if strings.HasPrefix(s, "######") {
			// H6
			md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToH6, Values: parseInline(trimHeaderChars(s))})
		} else if strings.HasPrefix(s, "#####") {
			// H5
			md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToH5, Values: parseInline(trimHeaderChars(s))})
		} else if strings.HasPrefix(s, "####") {
			// H4
			md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToH4, Values: parseInline(trimHeaderChars(s))})
		} else if strings.HasPrefix(s, "###") {
			// H3
			md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToH3, Values: parseInline(trimHeaderChars(s))})
		} else if strings.HasPrefix(s, "##") {
			// H2
			md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToH2, Values: parseInline(trimHeaderChars(s))})
		} else if strings.HasPrefix(s, "#") {
			// H1
			md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToH1, Values: parseInline(trimHeaderChars(s))})
		} else if s == "" {
			if 0 < len(buf) {
				// End of paragraph
				md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToP, Values: buf})
				buf = []Inline{}
			}
		} else if 0 < len(buf) && strings.Replace(s, "=", "", -1) == "" {
			// Last line is H1
			if 1 < len(buf) {
				md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToP, Values: buf[:len(buf)-1]})
			}
			md.Elements = append(md.Elements, MarkdownElement{ConverterFunc: c.ToH1, Values: buf[len(buf)-1:]})
			buf = []Inline{}
		} else if 0 < len(buf) && strings.Replace(s, "-", "", -1) == "" {
			// Last line is H2
{'fraction_non_alphanumeric': 0.17702147099132023, 'fraction_numerical': 0.012334399269072635, 'mean_word_length': 3.9648526077097506, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2913929', 'n_tokens_mistral': 1731, 'n_tokens_neox': 1569, 'n_words': 347}