<commit_before>package rss

import (
  "bytes"
  "errors"
  "fmt"
  "io/ioutil"
  "net/http"
  "strings"
  "time"
)

// Parse RSS or Atom data.
func Parse(data []byte) (*Feed, error) {

  if strings.Contains(string(data), "<rss") {
    return parseRSS2(data, database)
  } else if strings.Contains(string(data), "xmlns=\"http://purl.org/rss/1.0/\"") {
    return parseRSS1(data, database)
  } else {
    return parseAtom(data, database)
  }

  panic("Unreachable.")
}

// Fetch downloads and parses the RSS feed at the given URL
func Fetch(url string) (*Feed, error) {
  resp, err := http.Get(url)
  if err != nil {
    return nil, err
  }
  defer resp.Body.Close()

  body, err := ioutil.ReadAll(resp.Body)
  if err != nil {
    return nil, err
  }

  out, err := Parse(body)
  if err != nil {
    return nil, err
  }

  if out.Link == "" {
    out.Link = url
  }
	
	if out.Link != url {
		fmt.Printf("Warning: Feed with URL %q parsed to URL %q.\n", url, out.Link)
	}

  return out, nil
}

// Feed is the top-level structure.
type Feed struct {
	Nickname    string
  Title       string
  Description string
  Link        string
  Image       *Image
  Items       []*Item
  ItemMap     map[string]struct{}
  Refresh     time.Time
  Unread      uint32
}

// Update fetches any new items and updates f.
func (f *Feed) Update() error {

  // Check that we don't update too often.
  if f.Refresh.After(time.Now()) {
    return nil
  }

  if f.Link == "" {
    return errors.New("Error: feed has no URL.")
  }

  if f.ItemMap == nil {
    f.ItemMap = make(map[string]struct{})
	  for _, item := range f.Items {
	    if _, ok := f.ItemMap[item.ID]; !ok {
	      f.ItemMap[item.ID] = struct{}{}
	    }
	  }
  }

  update, err := Fetch(f.Link)
  if err != nil {
    return err
  }

  f.Refresh = update.Refresh
  f.Title = update.Title
  f.Description = update.Description

  for _, item := range update.Items {
    if _, ok := f.ItemMap[item.ID]; !ok {
      f.Items = append(f.Items, item)
      f.ItemMap[item.ID] = struct{}{}
      f.Unread++
    }
  }

  return nil
}

func (f *Feed) String() string {<commit_msg>Added distinction between feed link (website) and update URL (RSS link)<commit_after>package rss

import (
  "bytes"
  "errors"
  "fmt"
  "io/ioutil"
  "net/http"
  "strings"
  "time"
)

// Parse RSS or Atom data.
func Parse(data []byte) (*Feed, error) {

  if strings.Contains(string(data), "<rss") {
    return parseRSS2(data, database)
  } else if strings.Contains(string(data), "xmlns=\"http://purl.org/rss/1.0/\"") {
    return parseRSS1(data, database)
  } else {
    return parseAtom(data, database)
  }

  panic("Unreachable.")
}

// Fetch downloads and parses the RSS feed at the given URL
func Fetch(url string) (*Feed, error) {
  resp, err := http.Get(url)
  if err != nil {
    return nil, err
  }
  defer resp.Body.Close()

  body, err := ioutil.ReadAll(resp.Body)
  if err != nil {
    return nil, err
  }

  out, err := Parse(body)
  if err != nil {
    return nil, err
  }

  if out.Link == "" {
    out.Link = url
  }
	
	out.UpdateURL = url

  return out, nil
}

// Feed is the top-level structure.
type Feed struct {
	Nickname    string
  Title       string
  Description string
  Link        string
	UpdateURL   string
  Image       *Image
  Items       []*Item
  ItemMap     map[string]struct{}
  Refresh     time.Time
  Unread      uint32
}

// Update fetches any new items and updates f.
func (f *Feed) Update() error {

  // Check that we don't update too often.
  if f.Refresh.After(time.Now()) {
    return nil
  }

  if f.UpdateURL == "" {
    return errors.New("Error: feed has no URL.")
  }

  if f.ItemMap == nil {
    f.ItemMap = make(map[string]struct{})
	  for _, item := range f.Items {
	    if _, ok := f.ItemMap[item.ID]; !ok {
	      f.ItemMap[item.ID] = struct{}{}
	    }
	  }
  }

  update, err := Fetch(f.UpdateURL)
  if err != nil {
    return err
  }

  f.Refresh = update.Refresh
  f.Title = update.Title
  f.Description = update.Description

  for _, item := range update.Items {
    if _, ok := f.ItemMap[item.ID]; !ok {
      f.Items = append(f.Items, item)
      f.ItemMap[item.ID] = struct{}{}
      f.Unread++
    }
  }

  return nil
}

func (f *Feed) String() string {
{'fraction_non_alphanumeric': 0.14383723681097704, 'fraction_numerical': 0.0028388928317956, 'mean_word_length': 2.440195280716029, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6083997', 'n_tokens_mistral': 1568, 'n_tokens_neox': 1502, 'n_words': 424}