<commit_before>package main

import (
	"log"
	"math/rand"
	"os"

	"net/url"

	"strconv"

	"github.com/ChimeraCoder/anaconda"
	"github.com/dns-gh/tojson"
)

const (
	// TODO save only relevant information on tweets, the file coudl become too large at some point otherwise
	tweetsFilePath = "tweets.json"
)

var (
	searchTweetQueries = [...]string{
		"asteroids earth",
		"asteroids mars",
		"asteroids space",
		"asteroids nasa",
		"asteroids deadly",
		"asteroids watch danger",
		"asteroids end world",
	}
	maxRetweetBySearch    = 2
	maxFavoriteCountWatch = 2
)

// TODO factorize with load, merge and update
func loadTweets(path string) ([]anaconda.Tweet, error) {
	tweets := &[]anaconda.Tweet{}
	if _, err := os.Stat(path); os.IsNotExist(err) {
		tojson.Save(path, tweets)
	}
	err := tojson.Load(path, tweets)
	if err != nil {
		return nil, err
	}
	return *tweets, nil
}

func mergeTweets(previous, current []anaconda.Tweet) ([]anaconda.Tweet, []anaconda.Tweet) {
	merged := []anaconda.Tweet{}
	diff := []anaconda.Tweet{}
	added := map[int64]struct{}{}
	for _, v := range previous {
		added[v.Id] = struct{}{}
		merged = append(merged, v)
	}
	for _, v := range current {
		if _, ok := added[v.Id]; ok {
			continue
		}
		added[v.Id] = struct{}{}
		merged = append(merged, v)
		diff = append(diff, v)
	}
	return merged, diff
}

func updateTweets(path string, current []anaconda.Tweet) ([]anaconda.Tweet, error) {
	previous, err := loadTweets(path)
	if err != nil {
		return nil, err
	}
	merged, diff := mergeTweets(previous, current)
	tojson.Save(path, merged)
	return diff, nil
}

func getRelevantTweets() ([]anaconda.Tweet, error) {
	query := searchTweetQueries[rand.Intn(len(searchTweetQueries))]<commit_msg>[srb] change random seed at every launch<commit_after>package main

import (
	"log"
	"math/rand"
	"os"
	"time"

	"net/url"

	"strconv"

	"github.com/ChimeraCoder/anaconda"
	"github.com/dns-gh/tojson"
)

const (
	// TODO save only relevant information on tweets, the file coudl become too large at some point otherwise
	tweetsFilePath = "tweets.json"
)

var (
	searchTweetQueries = [...]string{
		"asteroids earth",
		"asteroids mars",
		"asteroids space",
		"asteroids nasa",
		"asteroids deadly",
		"asteroids watch danger",
		"asteroids end world",
	}
	maxRetweetBySearch    = 2
	maxFavoriteCountWatch = 2
)

// TODO factorize with load, merge and update
func loadTweets(path string) ([]anaconda.Tweet, error) {
	tweets := &[]anaconda.Tweet{}
	if _, err := os.Stat(path); os.IsNotExist(err) {
		tojson.Save(path, tweets)
	}
	err := tojson.Load(path, tweets)
	if err != nil {
		return nil, err
	}
	return *tweets, nil
}

func mergeTweets(previous, current []anaconda.Tweet) ([]anaconda.Tweet, []anaconda.Tweet) {
	merged := []anaconda.Tweet{}
	diff := []anaconda.Tweet{}
	added := map[int64]struct{}{}
	for _, v := range previous {
		added[v.Id] = struct{}{}
		merged = append(merged, v)
	}
	for _, v := range current {
		if _, ok := added[v.Id]; ok {
			continue
		}
		added[v.Id] = struct{}{}
		merged = append(merged, v)
		diff = append(diff, v)
	}
	return merged, diff
}

func updateTweets(path string, current []anaconda.Tweet) ([]anaconda.Tweet, error) {
	previous, err := loadTweets(path)
	if err != nil {
		return nil, err
	}
	merged, diff := mergeTweets(previous, current)
	tojson.Save(path, merged)
	return diff, nil
}

func getRelevantTweets() ([]anaconda.Tweet, error) {
	query := searchTweetQueries[rand.New(rand.NewSource(time.Now().UnixNano())).Intn(len(searchTweetQueries))]
{'fraction_non_alphanumeric': 0.15296803652968036, 'fraction_numerical': 0.00228310502283105, 'mean_word_length': 4.17725258493353, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3073508', 'n_tokens_mistral': 1415, 'n_tokens_neox': 1341, 'n_words': 320}