<commit_before>package mboxrd

import (
	"fmt"
	"log"
	"regexp"
	"strings"
	"time"
)

const timestampFormat = "060102150405"

var (
	loc *time.Location
)

func init() {
	var err error
	loc, err = time.LoadLocation("UTC")
	if err != nil {
		log.Fatal(err)
	}
}

func TimeNorm(line string, errors chan error) (string, error) {
	t, er := time.Parse("Mon, 2 Jan 2006 15:04:05 -0700", line)
	if er == nil {
		return t.In(loc).Format(timestampFormat), nil
	}

	t, er = time.Parse("Mon, 2 Jan 2006 15:04:05 -0700 (MST)", line)
	if er == nil {
		return t.In(loc).Format(timestampFormat), nil
	}

	t, er = time.Parse("2 Jan 2006 15:04:05 -0700", line)
	if er == nil {
		return t.In(loc).Format(timestampFormat), nil
	}

	return "", er
}

// NameFromTimeUser returns a closed function used to extract
// a message file name based on the message timestamp and sender's
// username part of the email.
//
// It is an example on how to construct the file name from multiple
// headers.
func NameFromTimeUser(format string, errors chan error) ByLineName {
	const (
		datePrefix = "Date: "
		fromPrefix = "From: "
		headPrefix = "From "
	)

	var (
		ts, fr, hd string
		fromRE = regexp.MustCompile("(.*<)?(.*)(@.*)")
		headRE = regexp.MustCompile("^From (.*)(@.*)")
	)

	return func(line string, errors chan error) string {
		var er error

		if ts == "" && strings.HasPrefix(line, datePrefix){
			ts, er = TimeNorm(strings.TrimPrefix(line, datePrefix), errors)
			if er != nil {<commit_msg>Case insensitive Date line header<commit_after>package mboxrd

import (
	"fmt"
	"log"
	"regexp"
	"strings"
	"time"
)

const timestampFormat = "060102150405"

var (
	loc *time.Location
)

func init() {
	var err error
	loc, err = time.LoadLocation("UTC")
	if err != nil {
		log.Fatal(err)
	}
}

func TimeNorm(line string, errors chan error) (string, error) {
	t, er := time.Parse("Mon, 2 Jan 2006 15:04:05 -0700", line)
	if er == nil {
		return t.In(loc).Format(timestampFormat), nil
	}

	t, er = time.Parse("Mon, 2 Jan 2006 15:04:05 -0700 (MST)", line)
	if er == nil {
		return t.In(loc).Format(timestampFormat), nil
	}

	t, er = time.Parse("2 Jan 2006 15:04:05 -0700", line)
	if er == nil {
		return t.In(loc).Format(timestampFormat), nil
	}

	return "", er
}

// NameFromTimeUser returns a closed function used to extract
// a message file name based on the message timestamp and sender's
// username part of the email.
//
// It is an example on how to construct the file name from multiple
// headers.
func NameFromTimeUser(format string, errors chan error) ByLineName {
	const (
		fromPrefix = "From: "
		headPrefix = "From "
	)

	var (
		ts, fr, hd string
		fromRE = regexp.MustCompile(`(.*<)?(.*)(@.*)`)
		headRE = regexp.MustCompile(`^From (.*)(@.*)`)
		dateRE = regexp.MustCompile(`(?i)^date: (.*)`)
	)

	return func(line string, errors chan error) string {
		var er error

		if ts == "" && dateRE.MatchString(line) {
			parsedTS := dateRE.FindStringSubmatch(line)
			if parsedTS == nil {
				errors <- fmt.Errorf(
					"Failed to parse the timestamp from the line %q",
					line)
				return ""
			}

			ts, er = TimeNorm(parsedTS[1], errors)
			if er != nil {
{'fraction_non_alphanumeric': 0.15027061445399553, 'fraction_numerical': 0.036612543775867555, 'mean_word_length': 3.4757834757834756, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4353094', 'n_tokens_mistral': 1276, 'n_tokens_neox': 1114, 'n_words': 348}