<commit_before>package main

import (
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"testing"
	"time"
)

const testContent string = `# This is some content
And some more content

## It was the best of times.

### It was the worst of times.
<NAME> <NAME>`

var testHTML []string = []string{
	"<h1>This is some content</h1>",
	"<p>And some more content</p>",
	"<h2>It was the best of times.</h2>",
	"<h3>It was the worst of times.</h3>",
	"<p><NAME> <NAME></p>",
}

func testOnePost(t *testing.T, title, date, tags string, includePostHeaderLine bool,
	content string) {

	t.Logf("Testing title: %s, date: %s, tags: \"%s\", postHeaderLine: %s",
		title, date, tags, strconv.FormatBool(includePostHeaderLine))

	var expectedTime time.Time
	expectedTags := make(map[string]bool)
	var err error
	validPost := true

	f, err := ioutil.TempFile("", "testPost")
	if err != nil {
		t.Fatal("Creating test post:", err)
	}

	if title != "MISSING" {
		f.WriteString(title + "\n")
	} else {
		validPost = false
	}

	if date != "MISSING" {
		f.WriteString(date + "\n")
		expectedTime, err = time.Parse("1/2/06 3:04PM MST", strings.TrimSpace(date))
		if err != nil {
			validPost = false
		}
	} else {
		validPost = false
	}

	if tags != "MISSING" {
		f.WriteString(tags + "\n")

		tagsList := strings.Split(tags, ",")
		for _, tag := range tagsList {
			tag = strings.Title(strings.TrimSpace(tag))<commit_msg>Use PostTimeFormat instead of hardcoding<commit_after>package main

import (
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"testing"
	"time"
)

const testContent string = `# This is some content
And some more content

## It was the best of times.

### It was the worst of times.
<NAME> <NAME>`

var testHTML []string = []string{
	"<h1>This is some content</h1>",
	"<p>And some more content</p>",
	"<h2>It was the best of times.</h2>",
	"<h3>It was the worst of times.</h3>",
	"<p><NAME> <NAME></p>",
}

func testOnePost(t *testing.T, title, date, tags string, includePostHeaderLine bool,
	content string) {

	t.Logf("Testing title: %s, date: %s, tags: \"%s\", postHeaderLine: %s",
		title, date, tags, strconv.FormatBool(includePostHeaderLine))

	var expectedTime time.Time
	expectedTags := make(map[string]bool)
	var err error
	validPost := true

	f, err := ioutil.TempFile("", "testPost")
	if err != nil {
		t.Fatal("Creating test post:", err)
	}

	if title != "MISSING" {
		f.WriteString(title + "\n")
	} else {
		validPost = false
	}

	if date != "MISSING" {
		f.WriteString(date + "\n")
		expectedTime, err = time.Parse(PostTimeFormat, strings.TrimSpace(date))
		if err != nil {
			validPost = false
		}
	} else {
		validPost = false
	}

	if tags != "MISSING" {
		f.WriteString(tags + "\n")

		tagsList := strings.Split(tags, ",")
		for _, tag := range tagsList {
			tag = strings.Title(strings.TrimSpace(tag))
{'fraction_non_alphanumeric': 0.16900913562895292, 'fraction_numerical': 0.0066760365425158116, 'mean_word_length': 3.833616298811545, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4957613', 'n_tokens_mistral': 1128, 'n_tokens_neox': 1061, 'n_words': 297}