<commit_before>package main

import (
	"bytes"
	"fmt"
	"html/template"
	"io"
	"log"
	"strings"
	"time"

	"github.com/emersion/go-imap/client"
	message "github.com/emersion/go-message"
	"github.com/emersion/go-message/mail"
	"github.com/mmcdole/gofeed"
	"github.com/spf13/viper"
)

var mailTemplate = template.Must(template.New("mail").Parse(
	`<table>
	<tbody>
	<tr><td>
	<a href='{{ .Link }}'>{{ .Title }}</a> {{ .Author }}
	<hr>
	</td></tr>
	<tr><td>
	{{ .Content }}
	</td></tr>
	</tbody>
	</table>`))

type templatePayload struct {
	Link    string
	Title   string
	Author  string
	Content template.HTML
}

func FormatContent(item *gofeed.Item) (string, error) {
	var payload templatePayload

	payload.Link = item.Link
	payload.Title = item.Title

	if item.Author != nil {
		payload.Author = fmt.Sprintf("%s %s", item.Author.Name, item.Author.Email)
	}

	if len(item.Content) > 0 {
		payload.Content = template.HTML(item.Content)
	} else {
		payload.Content = template.HTML(item.Description)
	}

	var buffer bytes.Buffer

	err := mailTemplate.Execute(&buffer, payload)

	if err != nil {
		return "", err
	}

	return buffer.String(), nil
}

func NewMessage(item *gofeed.Item) (bytes.Buffer, error) {
	var b bytes.Buffer

	fromName := viper.GetString("imap.from.name")

	if item.Author != nil {
		fromName = fmt.Sprintf("%s %s", item.Author.Name, item.Author.Email)
	}

	from := []*mail.Address{{fromName, viper.GetString("imap.from.email")}}
	to := []*mail.Address{{viper.GetString("imap.to.name"), viper.GetString("imap.to.email")}}

	h := mail.NewHeader()
	h.SetContentType("multipart/alternative", nil)
	h.SetDate(*item.PublishedParsed)
	h.SetAddressList("From", from)
	h.SetAddressList("To", to)
	h.SetSubject(item.Title)

	// Create a new mail writer
	messageWriter, err := message.CreateWriter(&b, h.Header)
	defer messageWriter.Close()
	if err != nil {
		return b, err
	}

	htmlHeader := make(message.Header)
	htmlHeader.SetContentType("text/html", nil)
	htmlWriter, err := messageWriter.CreatePart(htmlHeader)
	defer htmlWriter.Close()
	if err != nil {
		return b, err
	}

	content, err := FormatContent(item)
	if err != nil {
		return b, err
	}

	io.WriteString(htmlWriter, content)

	return b, nil
}

func AppendNewItemsViaIMAP(items ItemsWithFolders) error {
	if viper.GetBool("debug") {
		log.Printf("Found %d new items", len(items))
	}

	hostPort := fmt.Sprintf("%s:%d", viper.GetString("imap.host"), viper.GetInt("imap.port"))
	c, err := client.DialTLS(hostPort, nil)
	if err != nil {
		return err
	}

	defer c.Logout()

	if err := c.Login(viper.GetString("imap.username"), viper.GetString("imap.password")); err != nil {
		return err
	}

	if viper.GetBool("debug") {
		log.Println("Logged in to IMAP")
	}

	for _, entry := range items {
		if entry.Item.PublishedParsed == nil {
			t := time.Now()
			entry.Item.PublishedParsed = &t
		}

		folderName := entry.Folder
		if viper.GetBool("imap.folder_capitalize") {
			folderName = strings.Title(folderName)
		}
		folder := fmt.Sprintf("%s/%s", viper.GetString("imap.folder_prefix"), folderName)

		_ = c.Create(folder)

		msg, err := NewMessage(entry.Item)
		if err != nil {
			return err
		}

		if viper.GetBool("debug") {
			log.Printf("Appending item to %s", folder)
		}

		literal := bytes.NewReader(msg.Bytes())
		err = c.Append(folder, []string{}, *entry.Item.PublishedParsed, literal)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Fix template identation<commit_after>package main

import (
	"bytes"
	"fmt"
	"html/template"
	"io"
	"log"
	"strings"
	"time"

	"github.com/emersion/go-imap/client"
	message "github.com/emersion/go-message"
	"github.com/emersion/go-message/mail"
	"github.com/mmcdole/gofeed"
	"github.com/spf13/viper"
)

var mailTemplate = template.Must(template.New("mail").Parse(
	`<table>
<tbody>
<tr><td>
<a href='{{ .Link }}'>{{ .Title }}</a> {{ .Author }}
<hr>
</td></tr>
<tr><td>
{{ .Content }}
</td></tr>
</tbody>
</table>`))

type templatePayload struct {
	Link    string
	Title   string
	Author  string
	Content template.HTML
}

func FormatContent(item *gofeed.Item) (string, error) {
	var payload templatePayload

	payload.Link = item.Link
	payload.Title = item.Title

	if item.Author != nil {
		payload.Author = fmt.Sprintf("%s %s", item.Author.Name, item.Author.Email)
	}

	if len(item.Content) > 0 {
		payload.Content = template.HTML(item.Content)
	} else {
		payload.Content = template.HTML(item.Description)
	}

	var buffer bytes.Buffer

	err := mailTemplate.Execute(&buffer, payload)

	if err != nil {
		return "", err
	}

	return buffer.String(), nil
}

func NewMessage(item *gofeed.Item) (bytes.Buffer, error) {
	var b bytes.Buffer

	fromName := viper.GetString("imap.from.name")

	if item.Author != nil {
		fromName = fmt.Sprintf("%s %s", item.Author.Name, item.Author.Email)
	}

	from := []*mail.Address{{fromName, viper.GetString("imap.from.email")}}
	to := []*mail.Address{{viper.GetString("imap.to.name"), viper.GetString("imap.to.email")}}

	h := mail.NewHeader()
	h.SetContentType("multipart/alternative", nil)
	h.SetDate(*item.PublishedParsed)
	h.SetAddressList("From", from)
	h.SetAddressList("To", to)
	h.SetSubject(item.Title)

	// Create a new mail writer
	messageWriter, err := message.CreateWriter(&b, h.Header)
	defer messageWriter.Close()
	if err != nil {
		return b, err
	}

	htmlHeader := make(message.Header)
	htmlHeader.SetContentType("text/html", nil)
	htmlWriter, err := messageWriter.CreatePart(htmlHeader)
	defer htmlWriter.Close()
	if err != nil {
		return b, err
	}

	content, err := FormatContent(item)
	if err != nil {
		return b, err
	}

	io.WriteString(htmlWriter, content)

	return b, nil
}

func AppendNewItemsViaIMAP(items ItemsWithFolders) error {
	if viper.GetBool("debug") {
		log.Printf("Found %d new items", len(items))
	}

	hostPort := fmt.Sprintf("%s:%d", viper.GetString("imap.host"), viper.GetInt("imap.port"))
	c, err := client.DialTLS(hostPort, nil)
	if err != nil {
		return err
	}

	defer c.Logout()

	if err := c.Login(viper.GetString("imap.username"), viper.GetString("imap.password")); err != nil {
		return err
	}

	if viper.GetBool("debug") {
		log.Println("Logged in to IMAP")
	}

	for _, entry := range items {
		if entry.Item.PublishedParsed == nil {
			t := time.Now()
			entry.Item.PublishedParsed = &t
		}

		folderName := entry.Folder
		if viper.GetBool("imap.folder_capitalize") {
			folderName = strings.Title(folderName)
		}
		folder := fmt.Sprintf("%s/%s", viper.GetString("imap.folder_prefix"), folderName)

		_ = c.Create(folder)

		msg, err := NewMessage(entry.Item)
		if err != nil {
			return err
		}

		if viper.GetBool("debug") {
			log.Printf("Appending item to %s", folder)
		}

		literal := bytes.NewReader(msg.Bytes())
		err = c.Append(folder, []string{}, *entry.Item.PublishedParsed, literal)
		if err != nil {
			return err
		}
	}

	return nil
}

{'fraction_non_alphanumeric': 0.16832261835184104, 'fraction_numerical': 0.0008766803039158387, 'mean_word_length': 3.9854333576110705, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '267190', 'n_tokens_mistral': 2650, 'n_tokens_neox': 2509, 'n_words': 520}