<commit_before>package feed

import (
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"strings"
	"time"

	"github.com/mmcdole/go-xpp"
)

type RSSFeed struct {
	Title               string
	Link                string
	Description         string
	Language            string
	Copyright           string
	ManagingEditor      string
	WebMaster           string
	PubDate             string
	PubDateParsed       time.Time
	LastBuildDate       string
	LastBuildDateParsed time.Time
	Categories          []string
	Generator           string
	Docs                string
	TTL                 string
	Image               RSSImage
	Rating              string
	SkipHours           []string
	SkipDays            []string
	Items               []*RSSItem
	Version             string
	Extensions          map[string]interface{}
}

type RSSItem struct {
	Title         string
	Link          string
	Description   string
	Author        string
	Categories    []string
	Comments      string
	Enclosure     RSSEnclosure
	Guid          RSSGuid
	PubDate       string
	PubDateParsed time.Time
	Source        string
	Extensions    map[string]interface{}
}

type RSSImage struct {
	URL    string
	Link   string
	Width  string
	Height string
}

type RSSEnclosure struct {
	URL    string
	Length string
	Type   string
}

type RSSGuid struct {
	Value       string
	IsPermalink string
}

func ParseRSSFeed(feed string) (rss *RSSFeed, err error) {
	fmt.Println("Parsing feed...")

	p := xpp.NewXMLPullParser(strings.NewReader(feed))

	tok, err := p.NextTag()
	if err != nil {
		return
	}

	return parseRoot(p)
}

func parseRoot(p *xpp.XMLPullParser) (rss *RSSFeed, err error) {

	if !p.Matches(xpp.StartTag, nil, "rss") &&
		!p.Matches(xpp.StartTag, nil, "RDF") {
		return nil, errors.New("Unexpected root element")
	}

	items := []*RSSItem{}
	ver = parseVersion(p)

	for {
		tok, err := p.NextTag()
		if err != nil {
			return nil, err
		}

		if tok == xpp.EndTag {
			break
		}

		if tok == xpp.StartTag {
			if p.Name == "channel" {
				rss, err = parseChannel(p)
				if err != nil {
					return nil, err
				}
			} else if p.Name == "item" {
				// Earlier versions of the RSS spec had "item" elements at the same
				// root level as "channel" elements.  We will merge these items
				// with any channel level items.
				item, err = parseItem(i)
				if err != nil {
					return nil, err
				}
				items = append(items, item)
			} else {
				// Skip any elements that are not "channel" or "item"
				p.Skip()
			}
		}
	}

	if !p.Matches(xpp.EndTag, nil, "rss") &&
		!p.Matches(xpp.EndTag, nil, "RDF") {
		return nil, errors.New("Expected root end tag")
	}

	if rss != nil {
		rss.Items = append(rss.Items, items...)
		rss.Version = ver
		return rss, nil
	} else {
		return nil, errors.New("No channel element found.")
	}
}

func parseChannel(t xml.StartElement, d *xml.Decoder) (*RSSFeed, error) {
	if !p.Matches(xpp.StartTag, nil, "channel") {
		return nil, errors.New("Expected channel start tag")
	}

	rss := &RSSFeed{}
	rss.Items = []*RSSItem{}

	for {
		tok, err := p.NextTag()
		if err != nil {
			return nil, err
		}

		if tok == xpp.EndTag {
			break
		}

		if tok == xpp.StartTag {
			if p.Name == "title" {
				title, err := p.NextText()
				if err != nil {
					return nil, err
				}
				rss.Title = title
			} else {
				// Skip any elements not part of the channel spec
				p.Skip()
			}
		}
	}

	if !p.Matches(xpp.EndTag, nil, "channel") {
		return nil, errors.New("Expected channel end tag")
	}

	return rss, nil
}

func parseItem(t xml.StartElement, d *xml.Decoder) (*RSSItem, error) {
	return &RSSItem{}, nil
}

func parseVersion(p *xpp.XMLPullParser) (ver string) {
	if p.Name == "rss" {
		ver = p.Attribute("version")
	} else if p.Name == "RDF" {
		if p.Space == "http://channel.netscape.com/rdf/simple/0.9/" {
			ver = "0.9"
		} else if p.Space == "http://purl.org/rss/1.0/" {
			ver = "1.0"
		}
	}
	return
}
<commit_msg>Start parsing more fields on item/channel<commit_after>package feed

import (
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/mmcdole/go-xpp"
)

type RSSFeed struct {
	Title               string
	Link                string
	Description         string
	Language            string
	Copyright           string
	ManagingEditor      string
	WebMaster           string
	PubDate             string
	PubDateParsed       time.Time
	LastBuildDate       string
	LastBuildDateParsed time.Time
	Categories          []string
	Generator           string
	Docs                string
	TTL                 string
	Image               RSSImage
	Rating              string
	SkipHours           []string
	SkipDays            []string
	Items               []*RSSItem
	Version             string
	Extensions          map[string]interface{}
}

type RSSItem struct {
	Title         string
	Link          string
	Description   string
	Author        string
	Categories    []string
	Comments      string
	Enclosure     RSSEnclosure
	Guid          RSSGuid
	PubDate       string
	PubDateParsed time.Time
	Source        string
	Extensions    map[string]interface{}
}

type RSSImage struct {
	URL    string
	Link   string
	Width  string
	Height string
}

type RSSEnclosure struct {
	URL    string
	Length string
	Type   string
}

type RSSGuid struct {
	Value       string
	IsPermalink string
}

func ParseRSSFeed(feed string) (rss *RSSFeed, err error) {
	p := xpp.NewXMLPullParser(strings.NewReader(feed))

	_, err = p.NextTag()
	if err != nil {
		return
	}

	return parseRoot(p)
}

func parseRoot(p *xpp.XMLPullParser) (rss *RSSFeed, err error) {

	if !p.Matches(xpp.StartTag, "rss") &&
		!p.Matches(xpp.StartTag, "RDF") {
		return nil, errors.New("Unexpected root element")
	}

	items := []*RSSItem{}
	ver := parseVersion(p)

	for {
		tok, err := p.NextTag()
		if err != nil {
			return nil, err
		}

		if tok == xpp.EndTag {
			break
		}

		if tok == xpp.StartTag {
			if p.Name == "channel" {
				rss, err = parseChannel(p)
				if err != nil {
					return nil, err
				}
			} else if p.Name == "item" {
				// Earlier versions of the RSS spec had "item" elements at the same
				// root level as "channel" elements.  We will merge these items
				// with any channel level items.
				item, err := parseItem(p)
				if err != nil {
					return nil, err
				}
				items = append(items, item)
			} else {
				// Skip any elements that are not "channel" or "item"
				p.Skip()
			}
		}
	}

	if !p.Matches(xpp.EndTag, "rss") &&
		!p.Matches(xpp.EndTag, "RDF") {
		return nil, errors.New("Expected root end tag")
	}

	if rss != nil {
		rss.Items = append(rss.Items, items...)
		rss.Version = ver
		return rss, nil
	} else {
		return nil, errors.New("No channel element found.")
	}
}

func parseChannel(p *xpp.XMLPullParser) (*RSSFeed, error) {
	if !p.Matches(xpp.StartTag, "channel") {
		return nil, errors.New("Expected channel start tag")
	}

	rss := &RSSFeed{}
	rss.Items = []*RSSItem{}

	for {
		tok, err := p.NextTag()
		if err != nil {
			return nil, err
		}

		if tok == xpp.EndTag {
			break
		}

		if tok == xpp.StartTag {
			if p.Name == "title" {
				title, err := p.NextText()
				if err != nil {
					return nil, err
				}
				rss.Title = title
			} else if p.Name == "description" {
				desc, err := p.NextText()
				if err != nil {
					return nil, err
				}
				rss.Description = desc
			} else if p.Name == "link" {
				link, err := p.NextText()
				if err != nil {
					return nil, err
				}
				rss.Link = link
			} else if p.Name == "language" {
				lang, err := p.NextText()
				if err != nil {
					return nil, err
				}
				rss.Language = lang
			} else if p.Name == "copyright" {
				copyright, err := p.NextText()
				if err != nil {
					return nil, err
				}
				rss.Copyright = copyright
			} else if p.Name == "managingEditor" {
				editor, err := p.NextText()
				if err != nil {
					return nil, err
				}
				rss.ManagingEditor = editor
			} else if p.Name == "webMaster" {
				web, err := p.NextText()
				if err != nil {
					return nil, err
				}
				rss.WebMaster = web
			} else if p.Name == "pubDate" {
				pub, err := p.NextText()
				if err != nil {
					return nil, err
				}
				rss.PubDate = pub
			} else if p.Name == "lastBuildDate" {
				build, err := p.NextText()
				if err != nil {
					return nil, err
				}
				rss.LastBuildDate = build
			} else if p.Name == "generator" {
				gen, err := p.NextText()
				if err != nil {
					return nil, err
				}
				rss.Generator = gen
			} else if p.Name == "docs" {
				docs, err := p.NextText()
				if err != nil {
					return nil, err
				}
				rss.Docs = docs
			} else if p.Name == "ttl" {
				ttl, err := p.NextText()
				if err != nil {
					return nil, err
				}
				rss.TTL = ttl
			} else if p.Name == "rating" {
				rating, err := p.NextText()
				if err != nil {
					return nil, err
				}
				rss.Rating = rating
			} else {
				// Skip any elements not part of the channel spec
				p.Skip()
			}
		}
	}

	if !p.Matches(xpp.EndTag, "channel") {
		return nil, errors.New("Expected channel end tag")
	}

	return rss, nil
}

func parseItem(p *xpp.XMLPullParser) (*RSSItem, error) {
	if !p.Matches(xpp.StartTag, "item") {
		return nil, errors.New("Expected item start tag")
	}

	item := &RSSItem{}

	for {
		tok, err := p.NextTag()
		if err != nil {
			return nil, err
		}

		if tok == xpp.EndTag {
			break
		}

		if tok == xpp.StartTag {
			if p.Name == "title" {
				title, err := p.NextText()
				if err != nil {
					return nil, err
				}
				item.Title = title
			} else if p.Name == "description" {
				desc, err := p.NextText()
				if err != nil {
					return nil, err
				}
				item.Description = desc
			} else if p.Name == "link" {
				link, err := p.NextText()
				if err != nil {
					return nil, err
				}
				item.Link = link
			} else if p.Name == "author" {
				author, err := p.NextText()
				if err != nil {
					return nil, err
				}
				item.Author = author
			} else if p.Name == "comments" {
				comments, err := p.NextText()
				if err != nil {
					return nil, err
				}
				item.Comments = comments
			} else if p.Name == "pubDate" {
				pub, err := p.NextText()
				if err != nil {
					return nil, err
				}
				item.PubDate = pub
			} else if p.Name == "source" {
				source, err := p.NextText()
				if err != nil {
					return nil, err
				}
				item.Source = source
			} else {
				// Skip any elements not part of the channel spec
				p.Skip()
			}
		}
	}

	if !p.Matches(xpp.EndTag, "item") {
		return nil, errors.New("Expected item end tag")
	}

	return item, nil
}

func parseVersion(p *xpp.XMLPullParser) (ver string) {
	if p.Name == "rss" {
		ver = p.Attribute("version")
		if ver == "" {
			ver = "2.0"
		}
	} else if p.Name == "RDF" {
		ns := p.Attribute("xmlns")
		if ns == "http://channel.netscape.com/rdf/simple/0.9/" {
			ver = "0.9"
		} else if ns == "http://purl.org/rss/1.0/" {
			ver = "1.0"
		}
	}
	return
}

{'fraction_non_alphanumeric': 0.1269166820617033, 'fraction_numerical': 0.001662663957140218, 'mean_word_length': 2.1138912855910266, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1226830', 'n_tokens_mistral': 4471, 'n_tokens_neox': 3777, 'n_words': 1088}