<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"strings"
	"time"
)

type Post struct {
	SourcePath string
	Title      string
	Timestamp  time.Time
	Tags       []string
	Content    []byte
}

// NewPost reads a post from disk and returns a Post containing its data.
// If readContent is false, only the header of the post is read.
// The post format is:
// Title
// Date/Time
// Tags
//
// Markdown Content
func NewPost(filePath string, readContent bool) (p *Post, err error) {
	f, err := os.Open(filePath)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	p = &Post{SourcePath: filePath}

	reader := bufio.NewReader(f)

	line, err := reader.ReadString('\n')
	if err != nil {
		return
	}
	p.Title = string(line[0 : len(line)-1])<commit_msg>Add UrlTitle creation to Post.<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"strings"
	"time"
	"unicode"
)

type Post struct {
	SourcePath string
	Title      string
	UrlTitle   string
	Timestamp  time.Time
	Tags       []string
	Content    []byte
}

func urlTitleMap(r rune) rune {
	switch {
	case unicode.IsLetter(r):
		return unicode.ToLower(r)

	case unicode.IsDigit(r):
		return r

	case unicode.IsSpace(r):
		return '-'

	default:
		return -1
	}
}

func MakeUrlTitle(s string) string {
	return strings.Map(urlTitleMap, s)
}

// NewPost reads a post from disk and returns a Post containing its data.
// If readContent is false, only the header of the post is read.
// The post format is:
// Title
// Date/Time
// Tags
//
// Markdown Content
func NewPost(filePath string, readContent bool) (p *Post, err error) {
	f, err := os.Open(filePath)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	p = &Post{SourcePath: filePath}

	reader := bufio.NewReader(f)

	line, err := reader.ReadString('\n')
	if err != nil {
		return
	}
	p.Title = string(line[0 : len(line)-1])
	p.UrlTitle = MakeUrlTitle(p.Title)
{'fraction_non_alphanumeric': 0.12743549236440233, 'fraction_numerical': 0.0026329647182727752, 'mean_word_length': 3.1758241758241756, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3641259', 'n_tokens_mistral': 709, 'n_tokens_neox': 687, 'n_words': 216}