<commit_before>// this file contains a pre-processor to pull some stuff out of the markdown file before parsing it

package gnosis

import (
	"bufio"
	"bytes"
	"html/template"
	"os"
)

type pageMetadata struct {
	Keywords map[string]bool
	Tags     map[string]bool
	Loaded   bool
	Page     []byte
}

var pdata pageMetadata

func (pdata *PageMetadata) lineIsTitle(line []byte) bool {
	finalLength := len(line)
	i := 0

	// if the row doesn't start with tabs, spaces, or ='s
	if (data[i] != ' ' && data[i] != '=') && data[i] != '\t' {
		return false
	}

	// skip any spaces or tabs at the start
	for data[i] == ' ' || data[i] == '\t' {
		i++
	}

	// if the next item's not a =, bail out
	if data[i] != '=' {
		return false
	}

	// run through all of the ='s
	for data[i] == '=' {
		i++
	}

	if data[i] != ' ' && data[i] != '\t' && data[i] != '\n' {
		return false
	}

	//ditch all spaces after any ='s
	for data[i] == ' ' || data[i] == '\t' {
		i++
	}

	if finalLength == i+1 {
		return true
	} else {
		return false
	}
}

func (pdata *PageMetadata) checkMatch(input []byte, looking []byte, tracker []string) {
	// trim off any blank spaces at the start of the line
	value := bytes.Trim(line, " \t")

	if input[:len(looking)] == looking {
		// trim off the target from the []byte
		value = input[len(looking):]

		// trim spaces at the start and at the end
		value = bytes.Trim(value, " \t\n")

		if value[0] == ':' || value[0] == '=' {
			value = bytes.Trim(value, " \t\n=:")<commit_msg>fixed line references in lineIsTitle<commit_after>// this file contains a pre-processor to pull some stuff out of the markdown file before parsing it

package gnosis

import (
	"bufio"
	"bytes"
	"html/template"
	"os"
)

type PageMetadata struct {
	Keywords map[string]bool
	Tags     map[string]bool
	Loaded   bool
	Page     []byte
}

func (pdata *PageMetadata) lineIsTitle(line []byte) bool {
	finalLength := len(line)
	i := 0

	// if the row doesn't start with tabs, spaces, or ='s
	if (line[i] != ' ' && line[i] != '=') && line[i] != '\t' {
		return false
	}

	// skip any spaces or tabs at the start
	for line[i] == ' ' || line[i] == '\t' {
		i++
	}

	// if the next item's not a =, bail out
	if line[i] != '=' {
		return false
	}

	// run through all of the ='s
	for line[i] == '=' {
		i++
	}

	if line[i] != ' ' && line[i] != '\t' && line[i] != '\n' {
		return false
	}

	//ditch all spaces after any ='s
	for line[i] == ' ' || line[i] == '\t' {
		i++
	}

	if finalLength == i+1 {
		return true
	} else {
		return false
	}
}

func (pdata *PageMetadata) checkMatch(input []byte, looking []byte, tracker []string) {
	// trim off any blank spaces at the start of the line
	value := bytes.Trim(input, " \t")

	if input[:len(looking)] == looking {
		// trim off the target from the []byte
		value = input[len(looking):]

		// trim spaces at the start and at the end
		value = bytes.Trim(value, " \t\n")

		if value[0] == ':' || value[0] == '=' {
			value = bytes.Trim(value, " \t\n=:")
{'fraction_non_alphanumeric': 0.1771043771043771, 'fraction_numerical': 0.0026936026936026937, 'mean_word_length': 2.9247027741083222, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1814417', 'n_tokens_mistral': 1172, 'n_tokens_neox': 1101, 'n_words': 360}