<commit_before><commit_msg>Add basic stat crawler service implementation.<commit_after>package poller

import (
	"encoding/json"
	"github.com/zetafunction/steam-monster-game/messages"
	"github.com/zetafunction/steam-monster-game/steam"
	"log"
	"time"
)

type StatCrawler struct {
	api  *steam.APIService
	quit chan struct{}

	nonEndedUpdate chan int
	nonEnded       int

	invalidUpdate chan int
	invalid       int

	update chan []byte
}

func NewStatCrawler(api *steam.APIService, finder *RangeFinder) *StatCrawler {
	invalidUpdate := finder.SubscribeInvalid()
	nonEndedUpdate := finder.SubscribeNonEnded()
	return &StatCrawler{
		api,
		make(chan struct{}),
		nonEndedUpdate,
		<-nonEndedUpdate,
		invalidUpdate,
		<-invalidUpdate,
		make(chan []byte),
	}
}

func (c *StatCrawler) Start() {
	go func() {
		t := time.After(time.Second * 30)
		for {
			select {
			case <-t:
				func() {
					json, err := c.updateData()
					if err != nil {
						log.Print("stat crawler: update failed: ", err)
						return
					}
					c.update <- json
				}()
				t = time.After(time.Second * 30)
			case c.invalid = <-c.invalidUpdate:
			case c.nonEnded = <-c.nonEndedUpdate:
			}
		}
	}()
}

func (c *StatCrawler) Stop() {
	// TODO: This kind of gives the impression that things can be cleanly shut down.
	// Maybe just remove all the Stop() method receivers, since that really isn't the case.
	close(c.quit)
}

func (c *StatCrawler) GetUpdateChannel() <-chan []byte {
	return c.update
}

func (c *StatCrawler) updateData() ([]byte, error) {
	log.Print("stat crawler: updating from ", c.nonEnded, " to ", c.invalid)
	type update struct {
		id     int
		result *steam.GameDataResult
	}
	updates := make(chan update)
	requests := 0
	for i := c.nonEnded; i < c.invalid; i++ {
		go func(i int) {
			updates <- update{i, <-c.api.GetGameData(i)}
		}(i)
		requests++
	}
	m := make(map[int]*steam.GameDataResult)
	for requests > 0 {
		update := <-updates
		m[update.id] = update.result
		requests--
	}
	type statEntry struct {
		ID            int
		Level         uint32
		GameStart     uint32
		LevelStart    uint32
		ActivePlayers uint32
		Players       uint32
		Clicks        uint64
		AbilitiesUsed uint64
		ItemsUsed     uint64
	}
	var results []statEntry
	for i := c.nonEnded; i < c.invalid; i++ {
		if m[i].Err != nil {
			continue
		}
		r := m[i].Response
		if r.GetGameData().GetStatus() != messages.EMiniGameStatus_k_EMiniGameStatus_Running {
			continue
		}
		results = append(results, statEntry{
			ID:            i,
			Level:         r.GetGameData().GetLevel(),
			GameStart:     r.GetGameData().GetTimestampGameStart(),
			LevelStart:    r.GetGameData().GetTimestampLevelStart(),
			ActivePlayers: r.GetStats().GetNumActivePlayers(),
			Players:       r.GetStats().GetNumPlayers(),
			Clicks:        r.GetStats().GetNumClicks(),
			AbilitiesUsed: r.GetStats().GetNumAbilitiesActivated(),
			ItemsUsed:     r.GetStats().GetNumAbilityItemsActivated(),
		})
	}
	return json.Marshal(results)
}
{'fraction_non_alphanumeric': 0.12953020134228188, 'fraction_numerical': 0.00738255033557047, 'mean_word_length': 3.1634078212290504, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3776384', 'n_tokens_mistral': 1185, 'n_tokens_neox': 1036, 'n_words': 245}