<commit_before>// this file contains a pre-processor to pull some stuff out of the markdown file before parsing it

package goki

import (
	"bufio"
	"bytes"
	"io"
	"os"
	"sort"

	"github.com/JackKnifed/blackfriday"
)

type PageMetadata struct {
	Keywords  map[string]bool
	Topics    map[string]bool
	Authors   map[string]bool
	Page      []byte
	Title     string
	FileStats os.FileInfo
}

const (
	bodyHtmlFlags = 0 |
		blackfriday.HTML_USE_XHTML |
		blackfriday.HTML_USE_SMARTYPANTS |
		blackfriday.HTML_SMARTYPANTS_FRACTIONS |
		blackfriday.HTML_SMARTYPANTS_LATEX_DASHES |
		blackfriday.HTML_ALERT_BOXES

	bodyExtensions = 0 |
		blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
		blackfriday.EXTENSION_TABLES |
		blackfriday.EXTENSION_FENCED_CODE |
		blackfriday.EXTENSION_AUTOLINK |
		blackfriday.EXTENSION_STRIKETHROUGH |
		blackfriday.EXTENSION_SPACE_HEADERS |
		blackfriday.EXTENSION_AUTO_HEADER_IDS |
		blackfriday.EXTENSION_TITLEBLOCK |
		blackfriday.EXTENSION_ALERT_BOXES

	tocHtmlFlags = 0 |
		blackfriday.HTML_USE_XHTML |
		blackfriday.HTML_SMARTYPANTS_FRACTIONS |
		blackfriday.HTML_SMARTYPANTS_LATEX_DASHES |
		blackfriday.HTML_TOC |
		blackfriday.HTML_OMIT_CONTENTS

	tocExtensions = 0 |
		blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
		blackfriday.EXTENSION_TABLES |
		blackfriday.EXTENSION_FENCED_CODE |
		blackfriday.EXTENSION_AUTOLINK |
		blackfriday.EXTENSION_STRIKETHROUGH |
		blackfriday.EXTENSION_SPACE_HEADERS |
		blackfriday.EXTENSION_AUTO_HEADER_IDS |
		blackfriday.EXTENSION_TITLEBLOCK
)

// take a given line, and check it against every possible type of tag
func (pdata *PageMetadata) processMetadata(line []byte) {
	pdata.checkMatch(line, []byte("tag"), &pdata.Topics)
	pdata.checkMatch(line, []byte("topic"), &pdata.Topics)
	pdata.checkMatch(line, []byte("category"), &pdata.Topics)

	pdata.checkMatch(line, []byte("keyword"), &pdata.Keywords)
	pdata.checkMatch(line, []byte("meta"), &pdata.Keywords)

	pdata.checkMatch(line, []byte("author"), &pdata.Authors)
	pdata.checkMatch(line, []byte("maintainer"), &pdata.Authors)
}

func (pdata *PageMetadata) checkMatch(
	input []byte, looking []byte, tracker *map[string]bool) {
	// trim off any blank spaces at the start of the line
	input = bytes.ToLower(bytes.TrimSpace(input))
	looking = bytes.ToLower(bytes.TrimSpace(looking))

	if !bytes.HasPrefix(input, looking) {
		return
	}

	input = bytes.TrimSpace(bytes.TrimPrefix(input, looking))

	if input[0] != '=' && input[0] != ':' {
		return
	}

	input = bytes.TrimSpace(input[1:])

	if input[0] == '=' || input[0] == ':' {
		return
	}

	input = bytes.Replace(input, []byte("\t"), []byte(" "), -1)

	parts := bytes.Split(input, []byte(" "))
	var cleanParts [][]byte

	for _, piece := range parts {
		if len(piece) > 0 {
			cleanParts = append(cleanParts[:], piece)
		}
	}

	key := bytes.Join(cleanParts, []byte("-"))

	if *tracker != nil {
		(*tracker)[string(key)] = true
	} else {
		*tracker = map[string]bool{string(key): true}
	}
}

func (pdata *PageMetadata) readRestOfPage(r *bufio.Reader) error {
	// read the rest of the page
	var restOfPage []byte
	var err error

	for err == nil {
		// read a line, and then add it to pdata
		restOfPage, err = r.ReadBytes('\n')
		pdata.Page = append(pdata.Page, restOfPage...)
	}

	if err == io.EOF {
		return nil
	} else {
		return err
	}
}

func (pdata *PageMetadata) LoadPage(pageName string) error {
	// open the file
	f, err := os.Open(pageName)
	defer f.Close()
	reader := bufio.NewReader(f)
	if err != nil {<commit_msg>fixed a style complaint<commit_after>// this file contains a pre-processor to pull some stuff out of the markdown file before parsing it

package goki

import (
	"bufio"
	"bytes"
	"io"
	"os"
	"sort"

	"github.com/JackKnifed/blackfriday"
)

type PageMetadata struct {
	Keywords  map[string]bool
	Topics    map[string]bool
	Authors   map[string]bool
	Page      []byte
	Title     string
	FileStats os.FileInfo
}

const (
	bodyHtmlFlags = 0 |
		blackfriday.HTML_USE_XHTML |
		blackfriday.HTML_USE_SMARTYPANTS |
		blackfriday.HTML_SMARTYPANTS_FRACTIONS |
		blackfriday.HTML_SMARTYPANTS_LATEX_DASHES |
		blackfriday.HTML_ALERT_BOXES

	bodyExtensions = 0 |
		blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
		blackfriday.EXTENSION_TABLES |
		blackfriday.EXTENSION_FENCED_CODE |
		blackfriday.EXTENSION_AUTOLINK |
		blackfriday.EXTENSION_STRIKETHROUGH |
		blackfriday.EXTENSION_SPACE_HEADERS |
		blackfriday.EXTENSION_AUTO_HEADER_IDS |
		blackfriday.EXTENSION_TITLEBLOCK |
		blackfriday.EXTENSION_ALERT_BOXES

	tocHtmlFlags = 0 |
		blackfriday.HTML_USE_XHTML |
		blackfriday.HTML_SMARTYPANTS_FRACTIONS |
		blackfriday.HTML_SMARTYPANTS_LATEX_DASHES |
		blackfriday.HTML_TOC |
		blackfriday.HTML_OMIT_CONTENTS

	tocExtensions = 0 |
		blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
		blackfriday.EXTENSION_TABLES |
		blackfriday.EXTENSION_FENCED_CODE |
		blackfriday.EXTENSION_AUTOLINK |
		blackfriday.EXTENSION_STRIKETHROUGH |
		blackfriday.EXTENSION_SPACE_HEADERS |
		blackfriday.EXTENSION_AUTO_HEADER_IDS |
		blackfriday.EXTENSION_TITLEBLOCK
)

// take a given line, and check it against every possible type of tag
func (pdata *PageMetadata) processMetadata(line []byte) {
	pdata.checkMatch(line, []byte("tag"), &pdata.Topics)
	pdata.checkMatch(line, []byte("topic"), &pdata.Topics)
	pdata.checkMatch(line, []byte("category"), &pdata.Topics)

	pdata.checkMatch(line, []byte("keyword"), &pdata.Keywords)
	pdata.checkMatch(line, []byte("meta"), &pdata.Keywords)

	pdata.checkMatch(line, []byte("author"), &pdata.Authors)
	pdata.checkMatch(line, []byte("maintainer"), &pdata.Authors)
}

func (pdata *PageMetadata) checkMatch(
	input []byte, looking []byte, tracker *map[string]bool) {
	// trim off any blank spaces at the start of the line
	input = bytes.ToLower(bytes.TrimSpace(input))
	looking = bytes.ToLower(bytes.TrimSpace(looking))

	if !bytes.HasPrefix(input, looking) {
		return
	}

	input = bytes.TrimSpace(bytes.TrimPrefix(input, looking))

	if input[0] != '=' && input[0] != ':' {
		return
	}

	input = bytes.TrimSpace(input[1:])

	if input[0] == '=' || input[0] == ':' {
		return
	}

	input = bytes.Replace(input, []byte("\t"), []byte(" "), -1)

	parts := bytes.Split(input, []byte(" "))
	var cleanParts [][]byte

	for _, piece := range parts {
		if len(piece) > 0 {
			cleanParts = append(cleanParts[:], piece)
		}
	}

	key := bytes.Join(cleanParts, []byte("-"))

	if *tracker != nil {
		(*tracker)[string(key)] = true
	} else {
		*tracker = map[string]bool{string(key): true}
	}
}

func (pdata *PageMetadata) readRestOfPage(r *bufio.Reader) error {
	// read the rest of the page
	var restOfPage []byte
	var err error

	for err == nil {
		// read a line, and then add it to pdata
		restOfPage, err = r.ReadBytes('\n')
		pdata.Page = append(pdata.Page, restOfPage...)
	}

	if err == io.EOF {
		return nil
	}
	return err
}

func (pdata *PageMetadata) LoadPage(pageName string) error {
	// open the file
	f, err := os.Open(pageName)
	defer f.Close()
	reader := bufio.NewReader(f)
	if err != nil {
{'fraction_non_alphanumeric': 0.12799540823647582, 'fraction_numerical': 0.003156837422872722, 'mean_word_length': 4.625504439063761, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3541448', 'n_tokens_mistral': 2752, 'n_tokens_neox': 2628, 'n_words': 512}