<commit_before>package main

import (
	"html/template"
	"net/http"
	"os"
	"regexp"

	"log"

	"strings"

	"time"

	"strconv"

	"github.com/justinas/alice" // Middleware chaining
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
)

var wikiDir string
var tagDir string
var pubDir string
var ekey []byte

var encryptionFlag = []byte("ENCRYPTED")

var specialDir []string

type basePage struct {
	Title string
	Nav   nav
}
type wikiPage struct {
	Body      template.HTML
	Tags      string
	TagArray  []string
	Created   string
	Modified  string
	Published bool
	Encrypted bool
	basePage
}
type searchPage struct {
	basePage
	Results []QueryResults
}

func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}
func getPDFFilename(folder, name string) string {
	return folder + name
}

func getWikiFilename(folder, name string) string {
	return folder + name + ".md"
}

func getWikiTagsFilename(name string) string {
	return tagDir + name
}
func getWikiPubFilename(name string) string {
	return pubDir + name
}
func (p *wikiPage) save(s storage) error {
	var err error
	filename := getWikiFilename(wikiDir, p.Title)
	body := []byte(p.Body)
	if p.Encrypted {
		body, err = encrypt(body, ekey)
		if err != nil {
			return err
		}
		body = append(encryptionFlag, body...)
	}
	err = s.storeFile(filename, body)
	if err != nil {
		return err
	}

	tagsfile := getWikiTagsFilename(p.Title)
	err = s.storeFile(tagsfile, []byte(p.Tags))
	if err != nil {
		return err
	}

	log.Printf("Pub flag %v\n", p.Published)
	pubfile := getWikiPubFilename(p.Title)
	if p.Published {
		log.Printf("Saving %v\n", pubfile)
		err = s.storeFile(pubfile, nil)
		if err != nil {
			return err
		}

	} else {
		log.Printf("Removing %v\n", pubfile)
		// Only return an error if something other than file doesnt exit
		// We expect this fail to not exist most of the time but we dont know if
		// we don't try
		if err = s.deleteFile(pubfile); !os.IsNotExist(err) {
			return err
		}
	}

	return nil
}

func convertMarkdown(page *wikiPage, err error) (*wikiPage, error) {
	if err != nil {
		return page, err
	}
	p := bluemonday.UGCPolicy()
	p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code")

	page.Body = template.HTML(regexp.MustCompile("\r\n").ReplaceAllString(string(page.Body), "\n"))

	unsafe := blackfriday.Run([]byte(page.Body),
		blackfriday.WithExtensions(blackfriday.CommonExtensions|blackfriday.HardLineBreak),
	//blackfriday.WithRenderer(blackfriday.NewHTMLRenderer(
	//	blackfriday.HTMLRendererParameters{
	//		Flags: blackfriday.CommonHTMLFlags,
	//	}))
	)

	page.Body = template.HTML(p.SanitizeBytes(unsafe))
	return page, nil

}
func viewHandler(w http.ResponseWriter, r *http.Request, p *wikiPage, s storage) {
	p, err := convertMarkdown(s.getPage(p))
	if err != nil {
		p, err = s.checkForPDF(p)
		if err != nil {
			http.Redirect(w, r, "/wiki/edit/"+p.Title, http.StatusFound)
			return
		}
	} else {
		p.Body = template.HTML(parseWikiWords([]byte(p.Body)))
	}

	renderTemplate(w, "view", p)
}<commit_msg>Switch on IDs for headings<commit_after>package main

import (
	"html/template"
	"net/http"
	"os"
	"regexp"

	"log"

	"strings"

	"time"

	"strconv"

	"github.com/justinas/alice" // Middleware chaining
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
)

var wikiDir string
var tagDir string
var pubDir string
var ekey []byte

var encryptionFlag = []byte("ENCRYPTED")

var specialDir []string

type basePage struct {
	Title string
	Nav   nav
}
type wikiPage struct {
	Body      template.HTML
	Tags      string
	TagArray  []string
	Created   string
	Modified  string
	Published bool
	Encrypted bool
	basePage
}
type searchPage struct {
	basePage
	Results []QueryResults
}

func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}
func getPDFFilename(folder, name string) string {
	return folder + name
}

func getWikiFilename(folder, name string) string {
	return folder + name + ".md"
}

func getWikiTagsFilename(name string) string {
	return tagDir + name
}
func getWikiPubFilename(name string) string {
	return pubDir + name
}
func (p *wikiPage) save(s storage) error {
	var err error
	filename := getWikiFilename(wikiDir, p.Title)
	body := []byte(p.Body)
	if p.Encrypted {
		body, err = encrypt(body, ekey)
		if err != nil {
			return err
		}
		body = append(encryptionFlag, body...)
	}
	err = s.storeFile(filename, body)
	if err != nil {
		return err
	}

	tagsfile := getWikiTagsFilename(p.Title)
	err = s.storeFile(tagsfile, []byte(p.Tags))
	if err != nil {
		return err
	}

	log.Printf("Pub flag %v\n", p.Published)
	pubfile := getWikiPubFilename(p.Title)
	if p.Published {
		log.Printf("Saving %v\n", pubfile)
		err = s.storeFile(pubfile, nil)
		if err != nil {
			return err
		}

	} else {
		log.Printf("Removing %v\n", pubfile)
		// Only return an error if something other than file doesnt exit
		// We expect this fail to not exist most of the time but we dont know if
		// we don't try
		if err = s.deleteFile(pubfile); !os.IsNotExist(err) {
			return err
		}
	}

	return nil
}

func convertMarkdown(page *wikiPage, err error) (*wikiPage, error) {
	if err != nil {
		return page, err
	}
	p := bluemonday.UGCPolicy()
	p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code")

	page.Body = template.HTML(regexp.MustCompile("\r\n").ReplaceAllString(string(page.Body), "\n"))

	unsafe := blackfriday.Run([]byte(page.Body),
		blackfriday.WithExtensions(
			blackfriday.CommonExtensions|
				blackfriday.HardLineBreak|
				blackfriday.HeadingIDs|
				blackfriday.AutoHeadingIDs,
		),
	)

	page.Body = template.HTML(p.SanitizeBytes(unsafe))
	return page, nil

}
func viewHandler(w http.ResponseWriter, r *http.Request, p *wikiPage, s storage) {
	p, err := convertMarkdown(s.getPage(p))
	if err != nil {
		p, err = s.checkForPDF(p)
		if err != nil {
			http.Redirect(w, r, "/wiki/edit/"+p.Title, http.StatusFound)
			return
		}
	} else {
		p.Body = template.HTML(parseWikiWords([]byte(p.Body)))
	}

	renderTemplate(w, "view", p)
}
{'fraction_non_alphanumeric': 0.13279445727482678, 'fraction_numerical': 0.0006598482349059716, 'mean_word_length': 3.858173076923077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1097978', 'n_tokens_mistral': 2347, 'n_tokens_neox': 2208, 'n_words': 550}