<commit_before>// this file contains a pre-processor to pull some stuff out of the markdown file before parsing it

package gnosis

import (
	"bufio"
	"bytes"
	"errors"
	"html/template"
	"os"
)

type PageMetadata struct {
	Keywords map[string]bool
	Tags     map[string]bool
	Loaded   bool
	Page     []byte
}

func (pdata *PageMetadata) LoadPage(pageName string) error {
	f, err := os.Open(pageName)
	reader := bufio.NewReader(f)
	upperLine, fullLine, err := reader.ReadLine()

	// inspect the first line you read
	if err != nil {
		return err
	} else if !fullLine {
		return errors.New("first line I read wasn't a full line")
	} else if lineIsTitle(upperLine) {
		return errors.New("first line looks an awful lot like the underside of the title o.O")
	}

	lowerLine, fullLine, err := reader.ReadLine()

	// inspect the lower line
	if err != nil {
		return err
	} else if !fullLine {
		return errors.New("second line I read wasn't a full line")
	} else if lineIsTitle(lowerLine) {
		// if the second line is a title, read the rest of the page in
		// you don't have any metadata to work with here, move on
		upperLine.Append('\n')
		upperLine.Append(lowerLine)
		upperLine.Append('\n')

		_, err = reader.Read(lowerLine)
		if err != nil {
			return err
		}

		// you've successfully loaded the page - so return nothing
		pdata.Loaded = true
		return nil
	}

	// if you're at this point, the first line is metadata
	// you gotta process it and work with the next line
	// so let's just read through the file until we hit the title
	for !lineIsTitle(lowerLine) {<commit_msg>forgot the package reference in calls to lineIsTitle - fixed<commit_after>// this file contains a pre-processor to pull some stuff out of the markdown file before parsing it

package gnosis

import (
	"bufio"
	"bytes"
	"errors"
	"html/template"
	"os"
)

type PageMetadata struct {
	Keywords map[string]bool
	Tags     map[string]bool
	Loaded   bool
	Page     []byte
}

func (pdata *PageMetadata) LoadPage(pageName string) error {
	f, err := os.Open(pageName)
	reader := bufio.NewReader(f)
	upperLine, fullLine, err := reader.ReadLine()

	// inspect the first line you read
	if err != nil {
		return err
	} else if !fullLine {
		return errors.New("first line I read wasn't a full line")
	} else if pdata.lineIsTitle(upperLine) {
		return errors.New("first line looks an awful lot like the underside of the title o.O")
	}

	lowerLine, fullLine, err := reader.ReadLine()

	// inspect the lower line
	if err != nil {
		return err
	} else if !fullLine {
		return errors.New("second line I read wasn't a full line")
	} else if pdata.lineIsTitle(lowerLine) {
		// if the second line is a title, read the rest of the page in
		// you don't have any metadata to work with here, move on
		upperLine.Append('\n')
		upperLine.Append(lowerLine)
		upperLine.Append('\n')

		_, err = reader.Read(lowerLine)
		if err != nil {
			return err
		}

		// you've successfully loaded the page - so return nothing
		pdata.Loaded = true
		return nil
	}

	// if you're at this point, the first line is metadata
	// you gotta process it and work with the next line
	// so let's just read through the file until we hit the title
	for !pdata.lineIsTitle(lowerLine) {
{'fraction_non_alphanumeric': 0.09603991269098847, 'fraction_numerical': 0.0, 'mean_word_length': 3.4065934065934065, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5468028', 'n_tokens_mistral': 1071, 'n_tokens_neox': 1015, 'n_words': 412}