<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"path/filepath"
	"time"

	"github.com/nictuku/stardew-rocks/parser"
	"github.com/nictuku/stardew-rocks/stardb"
	"github.com/nictuku/stardew-rocks/view"

	"github.com/streadway/amqp"
)

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
	}
}

func wwwDir() string {
	home := os.Getenv("HOME")
	if home == "" {
		home = string(filepath.Separator)
	}
	return filepath.Clean(filepath.Join(home, "www"))
}

func main() {
	conn, err := amqp.Dial("amqp://guest:guest@amqp.stardew.rocks:5672/")
	failOnError(err, "Failed to connect to RabbitMQ")
	defer conn.Close()

	ch, err := conn.Channel()
	failOnError(err, "Failed to open a channel")
	defer ch.Close()
	for _, exc := range []string{"SaveGameInfo-1", "OtherFiles-1"} {
		err = ch.ExchangeDeclare(
			exc,      // name
			"fanout", // type
			false,    // durable
			false,    // auto-deleted
			false,    // internal
			false,    // no-wait
			nil,      // arguments
		)

		failOnError(err, "Failed to declare an exchange")
	}
	q, err := ch.QueueDeclare(
		"",    // name
		false, // durable
		false, // delete when usused
		true,  // exclusive
		false, // no-wait
		nil,   // arguments
	)
	failOnError(err, "Failed to declare a queue")

	err = ch.QueueBind(
		q.Name,         // queue name
		"",             // routing key
		"OtherFiles-1", // exchange
		false,
		nil)
	failOnError(err, "Failed to bind a queue")

	msgs, err := ch.Consume(
		q.Name, // queue
		"",     // consumer
		true,   // auto-ack
		false,  // exclusive
		false,  // no-local
		false,  // no-wait
		nil,    // args
	)
	failOnError(err, "Failed to register a consumer")

	count := 0

	farmMap := parser.LoadFarmMap()

	go func() {
		for d := range msgs {
			count++
			var reader io.Reader = bytes.NewReader(d.Body)
			// The content is usually gzip encoded by we don't have to worry about that.
			// Apparently rabbitMQ or the Go library will decompress it transparently.
			// d.ContentEncoding == "gzip" {
			saveGame, err := parser.ParseSaveGame(reader)
			if err != nil {
				log.Print("Error parsing saved game:", err)
				continue
			}
			if saveGame.Player.Name == "" {
				log.Print("Ignoring save with blank player name")
				continue
			}

			name := path.Base(path.Clean(saveGame.Player.Name)) // please don't hacko me mister

			ts := time.Now()

			farm, _, err := stardb.FindFarm(stardb.FarmCollection, saveGame.UniqueIDForThisGame, saveGame.Player.Name, saveGame.Player.FarmName)
			if err != nil {
				log.Print("Error fetching farm ID:", err)
				continue
			}

			// DEPRECATED filesystem write.
			//
			// Write the save game, then write the screenshot.
			// TODO: deal with races and conflicts.
			saveFile := path.Join(wwwDir(), "saveGames", fmt.Sprintf("%v-%d.xml", name, ts.Unix()))
			sf, err := os.OpenFile(saveFile, os.O_CREATE|os.O_WRONLY, 0666)
			if err != nil {
				log.Printf("Error opening saveGames %v: %v", saveFile, err)
				continue
			}
			if _, err := sf.Write(d.Body); err != nil {
				log.Printf("Failed to write save file at %v: %v", saveFile, err)
				continue
			} else {
				log.Printf("Wrote saveGame file %v", saveFile)
			}
			sf.Close()

			// GridFS XML save file write.
			if err := stardb.WriteSaveFile(farm, d.Body, ts); err != nil {
				log.Print("write save file:", err)
				continue
			}
			// The save file is the most critical and it's been updated, so we should be fine.
			if err := stardb.UpdateFarmTime(stardb.FarmCollection, farm.ID, ts); err != nil {
				log.Print("update farm time:", err)<commit_msg>mongodb web not yet working, but getting there<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"path/filepath"
	"time"

	"github.com/nictuku/stardew-rocks/parser"
	"github.com/nictuku/stardew-rocks/stardb"
	"github.com/nictuku/stardew-rocks/view"

	"github.com/streadway/amqp"
)

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
	}
}

func wwwDir() string {
	home := os.Getenv("HOME")
	if home == "" {
		home = string(filepath.Separator)
	}
	return filepath.Clean(filepath.Join(home, "www"))
}

func main() {
	conn, err := amqp.Dial("amqp://guest:guest@amqp.stardew.rocks:5672/")
	failOnError(err, "Failed to connect to RabbitMQ")
	defer conn.Close()

	ch, err := conn.Channel()
	failOnError(err, "Failed to open a channel")
	defer ch.Close()
	for _, exc := range []string{"SaveGameInfo-1", "OtherFiles-1"} {
		err = ch.ExchangeDeclare(
			exc,      // name
			"fanout", // type
			false,    // durable
			false,    // auto-deleted
			false,    // internal
			false,    // no-wait
			nil,      // arguments
		)

		failOnError(err, "Failed to declare an exchange")
	}
	q, err := ch.QueueDeclare(
		"",    // name
		false, // durable
		false, // delete when usused
		true,  // exclusive
		false, // no-wait
		nil,   // arguments
	)
	failOnError(err, "Failed to declare a queue")

	err = ch.QueueBind(
		q.Name,         // queue name
		"",             // routing key
		"OtherFiles-1", // exchange
		false,
		nil)
	failOnError(err, "Failed to bind a queue")

	msgs, err := ch.Consume(
		q.Name, // queue
		"",     // consumer
		true,   // auto-ack
		false,  // exclusive
		false,  // no-local
		false,  // no-wait
		nil,    // args
	)
	failOnError(err, "Failed to register a consumer")

	count := 0

	farmMap := parser.LoadFarmMap()

	go func() {
		for d := range msgs {
			count++
			var reader io.Reader = bytes.NewReader(d.Body)
			// The content is usually gzip encoded by we don't have to worry about that.
			// Apparently rabbitMQ or the Go library will decompress it transparently.
			// d.ContentEncoding == "gzip" {
			saveGame, err := parser.ParseSaveGame(reader)
			if err != nil {
				log.Print("Error parsing saved game:", err)
				continue
			}
			if saveGame.Player.Name == "" {
				log.Print("Ignoring save with blank player name")
				continue
			}

			name := path.Base(path.Clean(saveGame.Player.Name)) // please don't hacko me mister

			ts := time.Now()

			farm, _, err := stardb.FindFarm(stardb.FarmCollection, saveGame.UniqueIDForThisGame, saveGame.Player.Name, saveGame.Player.FarmName)
			if err != nil {
				log.Print("Error fetching farm ID:", err)
				continue
			}

			// DEPRECATED filesystem write.
			//
			// Write the save game, then write the screenshot.
			// TODO: deal with races and conflicts.
			saveFile := path.Join(wwwDir(), "saveGames", fmt.Sprintf("%v-%d.xml", name, ts.Unix()))
			sf, err := os.OpenFile(saveFile, os.O_CREATE|os.O_WRONLY, 0666)
			if err != nil {
				log.Printf("Error opening saveGames %v: %v", saveFile, err)
				continue
			}
			if _, err := sf.Write(d.Body); err != nil {
				log.Printf("Failed to write save file at %v: %v", saveFile, err)
				continue
			} else {
				log.Printf("Wrote saveGame file %v", saveFile)
			}
			sf.Close()

			// GridFS XML save file write.
			// TODO: broken saves (length 0)
			if err := stardb.WriteSaveFile(farm, d.Body, ts); err != nil {
				log.Print("write save file:", err)
				continue
			}
			// The save file is the most critical and it's been updated, so we should be fine.
			if err := stardb.UpdateFarmTime(stardb.FarmCollection, farm.ID, ts); err != nil {
				log.Print("update farm time:", err)
{'fraction_non_alphanumeric': 0.1489507612124537, 'fraction_numerical': 0.0034288849266218627, 'mean_word_length': 2.7998957790515893, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 2}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1968351', 'n_tokens_mistral': 2943, 'n_tokens_neox': 2591, 'n_words': 697}