<commit_before>package mlxscraper

import (
	"fmt"
	"log"
	"sort"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"

	"github.com/gbl08ma/disturbancesmlx/dataobjects"
	"github.com/mmcdole/gofeed"
)

// RSSScraper is an announcement scraper for the Metro de Lisboa website
// It reads the RSS feed from the official website
type RSSScraper struct {
	ticker         *time.Ticker
	stopChan       chan struct{}
	log            *log.Logger
	newAnnCallback func(announcement *dataobjects.Announcement)
	firstUpdate    bool
	fp             *gofeed.Parser
	announcements  []*dataobjects.Announcement

	URL     string
	Network *dataobjects.Network
	Period  time.Duration
}

// Begin starts the scraper
func (sc *RSSScraper) Begin(log *log.Logger,
	newAnnCallback func(announcement *dataobjects.Announcement)) {
	sc.stopChan = make(chan struct{})
	sc.ticker = time.NewTicker(sc.Period)
	sc.log = log
	sc.newAnnCallback = newAnnCallback
	sc.firstUpdate = true
	sc.fp = gofeed.NewParser()

	sc.log.Println("RSSScraper starting")
	sc.update()
	sc.firstUpdate = false
	sc.log.Println("RSSScraper completed first fetch")
	go sc.scrape()
}

// End stops the scraper
func (sc *RSSScraper) End() {
	sc.ticker.Stop()
	close(sc.stopChan)
}

func (sc *RSSScraper) copyAnnouncements() []*dataobjects.Announcement {
	c := make([]*dataobjects.Announcement, len(sc.announcements))
	for i, annPointer := range sc.announcements {
		ann := *annPointer
		c[i] = &ann
	}
	return c
}

func (sc *RSSScraper) scrape() {
	sc.update()
	sc.log.Println("RSSScraper completed second fetch")
	for {
		select {
		case <-sc.ticker.C:
			sc.update()
			sc.log.Println("RSSScraper fetch complete")
		case <-sc.stopChan:
			return
		}
	}
}

func (sc *RSSScraper) update() {
	feed, _ := sc.fp.ParseURL(sc.URL)

	announcements := []*dataobjects.Announcement{}
	for _, item := range feed.Items {
		ann := dataobjects.Announcement{
			Time:    *item.PublishedParsed,
			Network: sc.Network,
			Title:   item.Title,
			Body:    sc.adaptPostBody(item.Description),
			URL:     item.Link,
			Source:  "pt-ml-rss",
		}
		announcements = append(announcements, &ann)
	}

	sort.SliceStable(announcements, func(i, j int) bool {
		return announcements[i].Time.Before(announcements[j].Time)
	})

	for _, ann := range announcements {
		fmt.Println(ann.Title)
		fmt.Println(ann.Body)
		fmt.Println(ann.URL)
		fmt.Println(ann.Time)
	}
<commit_msg>Remove debug output<commit_after>package mlxscraper

import (
	"log"
	"sort"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"

	"github.com/gbl08ma/disturbancesmlx/dataobjects"
	"github.com/mmcdole/gofeed"
)

// RSSScraper is an announcement scraper for the Metro de Lisboa website
// It reads the RSS feed from the official website
type RSSScraper struct {
	ticker         *time.Ticker
	stopChan       chan struct{}
	log            *log.Logger
	newAnnCallback func(announcement *dataobjects.Announcement)
	firstUpdate    bool
	fp             *gofeed.Parser
	announcements  []*dataobjects.Announcement

	URL     string
	Network *dataobjects.Network
	Period  time.Duration
}

// Begin starts the scraper
func (sc *RSSScraper) Begin(log *log.Logger,
	newAnnCallback func(announcement *dataobjects.Announcement)) {
	sc.stopChan = make(chan struct{})
	sc.ticker = time.NewTicker(sc.Period)
	sc.log = log
	sc.newAnnCallback = newAnnCallback
	sc.firstUpdate = true
	sc.fp = gofeed.NewParser()

	sc.log.Println("RSSScraper starting")
	sc.update()
	sc.firstUpdate = false
	sc.log.Println("RSSScraper completed first fetch")
	go sc.scrape()
}

// End stops the scraper
func (sc *RSSScraper) End() {
	sc.ticker.Stop()
	close(sc.stopChan)
}

func (sc *RSSScraper) copyAnnouncements() []*dataobjects.Announcement {
	c := make([]*dataobjects.Announcement, len(sc.announcements))
	for i, annPointer := range sc.announcements {
		ann := *annPointer
		c[i] = &ann
	}
	return c
}

func (sc *RSSScraper) scrape() {
	sc.update()
	sc.log.Println("RSSScraper completed second fetch")
	for {
		select {
		case <-sc.ticker.C:
			sc.update()
			sc.log.Println("RSSScraper fetch complete")
		case <-sc.stopChan:
			return
		}
	}
}

func (sc *RSSScraper) update() {
	feed, _ := sc.fp.ParseURL(sc.URL)

	announcements := []*dataobjects.Announcement{}
	for _, item := range feed.Items {
		ann := dataobjects.Announcement{
			Time:    *item.PublishedParsed,
			Network: sc.Network,
			Title:   item.Title,
			Body:    sc.adaptPostBody(item.Description),
			URL:     item.Link,
			Source:  "pt-ml-rss",
		}
		announcements = append(announcements, &ann)
	}

	sort.SliceStable(announcements, func(i, j int) bool {
		return announcements[i].Time.Before(announcements[j].Time)
	})

{'fraction_non_alphanumeric': 0.12731581214993537, 'fraction_numerical': 0.0008616975441619991, 'mean_word_length': 3.8064182194616976, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '741315', 'n_tokens_mistral': 1828, 'n_tokens_neox': 1713, 'n_words': 363}