<commit_before>package main

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"reflect"
	"strconv"
	"time"

	"github.com/julienschmidt/httprouter"
)

const timeFormat = "2006-01-02"

type rate struct {
	Date  time.Time `json:"date"`
	Value float64   `json:"value"`
}

// MarshalJSON overrides marshal, prints shorter date string
func (r *rate) MarshalJSON() ([]byte, error) {
	type alias rate
	return json.Marshal(&struct {
		Date string `json:"date"`
		*alias
	}{
		Date:  r.Date.Format("2006-01-02"),
		alias: (*alias)(r),
	})
}

var lastRefresh time.Time
var maturities = []string{"1w", "2w", "1m", "2m", "3m", "6m", "9m", "12m"}
var retentions = []string{"week", "month", "three_months", "six_months", "year", "two_years", "six_years"}
var historyCache map[string][]rate
var influxCache map[string]map[string][]rate

// runs in go routine and takes care of refreshing the cache
func refreshCache() {
	historyCache = make(map[string][]rate)
	influxCache = make(map[string]map[string][]rate)

	for {
		if time.Since(lastRefresh) < time.Hour*24 {
			time.Sleep(time.Minute)
			continue
		}

		// refresh
		log.Println("refreshing history cache")
		for _, maturity := range maturities {
			// TODO: make path to files configureable
			file := fmt.Sprintf("../euribor-rates-%s.csv", maturity)
			historyCache[maturity] = parseFile(file)
		}
		log.Println("refreshing influx cache")
		for _, retention := range retentions {
			results := queryInflux(retention)
			cache := make(map[string][]rate)
			// fmt.Println(results.Values)
			// fmt.Println(reflect.TypeOf(results.Values).Kind())
			for _, value := range results.Values {
				// fmt.Println(value, reflect.TypeOf(value).Kind(), reflect.TypeOf(value))
				m, r, err := transformInfluxValueToRate(reflect.ValueOf(value).Interface().([]interface{}))
				if err != nil {
					log.Println("error converting influx value to rate")
					continue
				}
				if rates, ok := cache[m]; ok {
					rates = append(rates, r)
					cache[m] = rates
				} else {
					cache[m] = []rate{r}
				}
			}
			influxCache[retention] = cache
		}

		lastRefresh = time.Now()
	}
}
<commit_msg>Log when cache refresh is completed<commit_after>package main

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"reflect"
	"strconv"
	"time"

	"github.com/julienschmidt/httprouter"
)

const timeFormat = "2006-01-02"

type rate struct {
	Date  time.Time `json:"date"`
	Value float64   `json:"value"`
}

// MarshalJSON overrides marshal, prints shorter date string
func (r *rate) MarshalJSON() ([]byte, error) {
	type alias rate
	return json.Marshal(&struct {
		Date string `json:"date"`
		*alias
	}{
		Date:  r.Date.Format("2006-01-02"),
		alias: (*alias)(r),
	})
}

var lastRefresh time.Time
var maturities = []string{"1w", "2w", "1m", "2m", "3m", "6m", "9m", "12m"}
var retentions = []string{"week", "month", "three_months", "six_months", "year", "two_years", "six_years"}
var historyCache map[string][]rate
var influxCache map[string]map[string][]rate

// runs in go routine and takes care of refreshing the cache
func refreshCache() {
	historyCache = make(map[string][]rate)
	influxCache = make(map[string]map[string][]rate)

	for {
		if time.Since(lastRefresh) < time.Hour*24 {
			time.Sleep(time.Minute)
			continue
		}

		// refresh
		log.Println("refreshing history cache")
		for _, maturity := range maturities {
			// TODO: make path to files configureable
			file := fmt.Sprintf("../euribor-rates-%s.csv", maturity)
			historyCache[maturity] = parseFile(file)
		}
		log.Println("refreshing influx cache")
		for _, retention := range retentions {
			results := queryInflux(retention)
			cache := make(map[string][]rate)
			// fmt.Println(results.Values)
			// fmt.Println(reflect.TypeOf(results.Values).Kind())
			for _, value := range results.Values {
				// fmt.Println(value, reflect.TypeOf(value).Kind(), reflect.TypeOf(value))
				m, r, err := transformInfluxValueToRate(reflect.ValueOf(value).Interface().([]interface{}))
				if err != nil {
					log.Println("error converting influx value to rate")
					continue
				}
				if rates, ok := cache[m]; ok {
					rates = append(rates, r)
					cache[m] = rates
				} else {
					cache[m] = []rate{r}
				}
			}
			influxCache[retention] = cache
		}
		log.Println("cache refresh completed")

		lastRefresh = time.Now()
	}
}

{'fraction_non_alphanumeric': 0.1622676153316502, 'fraction_numerical': 0.013311911865962818, 'mean_word_length': 3.896629213483146, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4069013', 'n_tokens_mistral': 1748, 'n_tokens_neox': 1516, 'n_words': 352}