<commit_before>// Front-matter: http://jekyllrb.com/docs/frontmatter/
package space

import (
	"bufio"
	"bytes"
	"io"
	"regexp"
)

var (
	FRONT_MATTER = regexp.MustCompile(`---\s*`)
)

func FrontMatterParser(r *bufio.Reader) (contents, metedata *bytes.Buffer, err error) {
	var (
		// frontmatter: opened/closed
		opened, closed int
		buf            []byte
	)

	metedata = new(bytes.Buffer)
	contents = new(bytes.Buffer)

	for {
		// read line by line
		buf, err = r.ReadBytes('\n')
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, nil, err
		}

		if opened&closed == 0 {
			if FRONT_MATTER.Match(buf) {
				if opened == 0 {
					opened = 1
				} else {
					closed = 1
				}
			}

			if opened|closed == 1 {
				metedata.Write(buf)
			}
		} else {
			contents.Write(buf)
		}
	}

	if opened&closed == 0 {
		contents.WriteTo(metedata)
		contents = metedata
		metedata = nil
	}

	return contents, metedata, nil
}<commit_msg>improve parse front-matter<commit_after>// Front-matter: http://jekyllrb.com/docs/frontmatter/
package space

import (
	"bufio"
	"bytes"
	"io"
	"regexp"
)

var (
	FRONT_MATTER = regexp.MustCompile(`---\s*`)
)

func FrontMatterParser(r *bufio.Reader) (contents, metedata *bytes.Buffer, err error) {
	var (
		// check frontmatter
		// 0: no fm
		// 1: fm start
		// 2: fm end
		status int
		buf    []byte
	)

	metedata = new(bytes.Buffer)
	contents = new(bytes.Buffer)

	for {
		// read line by line
		buf, err = r.ReadBytes('\n')
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, nil, err
		}

		if status < 2 {
			if FRONT_MATTER.Match(buf) {
				if status == 0 {
					status = 1
				} else {
					status = 2
				}
			}

			if status > 0 {
				metedata.Write(buf)
			}
		} else {
			contents.Write(buf)
		}
	}

	if status == 0 {
		contents.WriteTo(metedata)
		contents = metedata
		metedata = nil
	}

	return contents, metedata, nil
}
{'fraction_non_alphanumeric': 0.14022140221402213, 'fraction_numerical': 0.00790722192936215, 'mean_word_length': 2.4509090909090907, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '69320', 'n_tokens_mistral': 875, 'n_tokens_neox': 733, 'n_words': 178}