<commit_before>package main

import (
	"errors"
	"fmt"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
	"io"
	"strings"
	"time"
)

var rootNodeAttr = "meta clear-block"

type Article struct {
	Content  []string
	metadata ArticleMetadata
}

type ArticleMetadata struct {
	Author string
	Title  string
	Tags   []string
	link   string
	Date   time.Time
}

func formatDate(dateString string) time.Time {
	parts := strings.Split(dateString, " ")
	parts[0] = parts[0][:3]
	parts[1] = parts[1][:len(parts[1])-3]
	newDateString := strings.Join(parts, " ")
	parsedDate, err := time.Parse("Jan 2 2006 at 3:04 PM", newDateString)
	if err != nil {
		fmt.Println(err)
		return time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC)
	}
	return parsedDate
}

func parseMetadataForArticles(article io.Reader) (metadata []ArticleMetadata, err error) {
	roots := getAllMatchingNodeAttrs(article, rootNodeAttr)
	for _, root := range roots {
		parsedMetadata, err := parseMetadata(root)
		if err == nil {
			metadata = append(metadata, parsedMetadata)
		} else {
			fmt.Println(err)
		}
	}
	return metadata, err
}

func parseMetadata(root *html.Node) (metadataFromPage ArticleMetadata, err error) {
	Link, err := getLink(root)
	if err != nil {
		fmt.Println(err)
		return metadataFromPage, err
	}
	author, err := getAuthor(root)
	if err != nil {
		fmt.Println(err)
		return metadataFromPage, err
	}
	title, err := getTitle(root)
	if err != nil {
		fmt.Println(err)
		return metadataFromPage, err
	}
	date, err := getDate(root)
	if err != nil {
		fmt.Println(err)
		return metadataFromPage, err
	}
	tags, err := getTags(root)
	if err != nil {
		fmt.Println(err)
		return metadataFromPage, err
	}
	//TODO: find better way to do this
	metadataFromPage.Author = author
	metadataFromPage.Title = title
	metadataFromPage.Date = date
	metadataFromPage.Tags = tags
	metadataFromPage.link = Link
	return metadataFromPage, err
}

func ProcessArticle(unparsedArticle io.Reader) (Article, error) {
	root := getFirstNode(unparsedArticle, rootNodeAttr)
	articleMetadata, _ := parseMetadata(root)
	article := Article{Content: getContent(root),
		metadata: articleMetadata}
	return article, nil
}

func getContent(node *html.Node) []string {
	contentNode := node.NextSibling.NextSibling.NextSibling.NextSibling.FirstChild
	content := []string{}
	content = getAllDataFromChildren(contentNode, content, atom.P)
	return content
}

func getTags(node *html.Node) ([]string, error) {
	tags, err := getListItems(node.NextSibling.NextSibling.FirstChild.NextSibling)
	return tags, err
}

func getLink(root *html.Node) (string, error) {
	var articleLink string<commit_msg>Cleaned up metadata parsing a bit<commit_after>package main

import (
	"errors"
	"fmt"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
	"io"
	"strings"
	"time"
)

var rootNodeAttr = "meta clear-block"

type Article struct {
	Content  []string
	metadata ArticleMetadata
}

type ArticleMetadata struct {
	Author string
	Title  string
	Tags   []string
	link   string
	Date   time.Time
}

func formatDate(dateString string) time.Time {
	parts := strings.Split(dateString, " ")
	parts[0] = parts[0][:3]
	parts[1] = parts[1][:len(parts[1])-3]
	newDateString := strings.Join(parts, " ")
	parsedDate, err := time.Parse("Jan 2 2006 at 3:04 PM", newDateString)
	if err != nil {
		fmt.Println(err)
		return time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC)
	}
	return parsedDate
}

func parseMetadataForArticles(article io.Reader) (metadata []ArticleMetadata, err error) {
	roots := getAllMatchingNodeAttrs(article, rootNodeAttr)
	for _, root := range roots {
		parsedMetadata, err := parseMetadata(root)
		if err == nil {
			metadata = append(metadata, parsedMetadata)
		} else {
			fmt.Println(err)
		}
	}
	return metadata, err
}

func parseMetadata(root *html.Node) (pageMetadata ArticleMetadata, err error) {
	pageMetadata.link, err = getLink(root)
	if err != nil {
		fmt.Println(err)
		return pageMetadata, err
	}

	pageMetadata.Author, err = getAuthor(root)
	if err != nil {
		fmt.Println(err)
		return pageMetadata, err
	}

	pageMetadata.Title, err = getTitle(root)
	if err != nil {
		fmt.Println(err)
		return pageMetadata, err
	}

	pageMetadata.Date, err = getDate(root)
	if err != nil {
		fmt.Println(err)
		return pageMetadata, err
	}

	pageMetadata.Tags, err = getTags(root)
	if err != nil {
		fmt.Println(err)
		return pageMetadata, err
	}

	return pageMetadata, err
}

func ProcessArticle(unparsedArticle io.Reader) (Article, error) {
	root := getFirstNode(unparsedArticle, rootNodeAttr)
	articleMetadata, _ := parseMetadata(root)
	article := Article{Content: getContent(root),
		metadata: articleMetadata}
	return article, nil
}

func getContent(node *html.Node) []string {
	contentNode := node.NextSibling.NextSibling.NextSibling.NextSibling.FirstChild
	content := []string{}
	content = getAllDataFromChildren(contentNode, content, atom.P)
	return content
}

func getTags(node *html.Node) ([]string, error) {
	tags, err := getListItems(node.NextSibling.NextSibling.FirstChild.NextSibling)
	return tags, err
}

func getLink(root *html.Node) (string, error) {
	var articleLink string
{'fraction_non_alphanumeric': 0.12365275328238291, 'fraction_numerical': 0.008622378992749364, 'mean_word_length': 4.565976008724101, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1712447', 'n_tokens_mistral': 1856, 'n_tokens_neox': 1783, 'n_words': 439}