<commit_before>package main

import (
	"time"
	"log"
	"strconv"
)

const (
	rate = int64(5)
)

type GorStat struct {
	statName	string
	latest		int
	min			int
	max			int

	currentTime int64
}

func NewGorStat(statName string) GorStat {
	s := new(GorStat)
	s.statName = statName
	s.latest = 0
	s.min = 0
	s.max = 0

	if Settings.stats {
		go s.reportStats()
	}
	return s
}

func (s *GorStat) Write(latest int) {
	if Settings.stats {
		if latest > s.max {
			s.max = latest
		}
		if latest < s.min {
			s.min = latest
		}
		s.latest = latest
	}
}

func (s *GorStat) String() string {
	return s.statName + ":" + strconv.Itoa(s.latest) + "," + strconv.Itoa(s.min) + "," + strconv.Itoa(s.max)
}

func (s *GorStat) reportStats() {
	if (time.Now().UnixNano() - s.currentTime) > (rate * time.Second.Nanoseconds()) {
		s.currentTime = time.Now().UnixNano()
		log.Println(s)
	}
}

<commit_msg>fixing pointers<commit_after>package main

import (
	"time"
	"log"
	"strconv"
)

const (
	rate = int64(5)
)

type GorStat struct {
	statName	string
	latest		int
	min			int
	max			int

	currentTime int64
}

func NewGorStat(statName string) (s *GorStat) {
	s := new(GorStat)
	s.statName = statName
	s.latest = 0
	s.min = 0
	s.max = 0

	if Settings.stats {
		go s.reportStats()
	}
	return
}

func (s *GorStat) Write(latest int) {
	if Settings.stats {
		if latest > s.max {
			s.max = latest
		}
		if latest < s.min {
			s.min = latest
		}
		s.latest = latest
	}
}

func (s *GorStat) String() string {
	return s.statName + ":" + strconv.Itoa(s.latest) + "," + strconv.Itoa(s.min) + "," + strconv.Itoa(s.max)
}

func (s *GorStat) reportStats() {
	if (time.Now().UnixNano() - s.currentTime) > (rate * time.Second.Nanoseconds()) {
		s.currentTime = time.Now().UnixNano()
		log.Println(s)
	}
}


{'fraction_non_alphanumeric': 0.1565856416054268, 'fraction_numerical': 0.009044657998869417, 'mean_word_length': 3.022727272727273, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5120347', 'n_tokens_mistral': 818, 'n_tokens_neox': 769, 'n_words': 167}