<commit_before>package main

import (
	"fmt"
	"bufio"
	"regexp"
	"strings"
	"time"
	"os"
)

type LogReader struct {
	Database          Database
	RegexAction       *regexp.Regexp
	RegexMessage      *regexp.Regexp
	RegexParseAction  *regexp.Regexp
	RegexParseMessage *regexp.Regexp
	Config            Config
}

func (lr *LogReader) LoadFile() bool {
	// Open the file
	f, err := os.Open(lr.Config.Location)

	if (err != nil) {
		return false
	}

	// Create Buffered Scanner
	scanner := bufio.NewScanner(f)

	// Loop over all found lines and analyse them
	for scanner.Scan() {
		line := scanner.Text()
		if lr.RegexAction.MatchString(line) == true {
			lr.ParseLine(line, true)
		} else if lr.RegexMessage.MatchString(line) == true {
			lr.ParseLine(line, false)
		} else {
			fmt.Printf("error reading line [%d]\n", lr.Database.Channel.LineCount)
		}
	}

	lr.Database.LastGenerated = time.Now().Unix()

	return true;
}

func (lr LogReader) ParseTime(inputDate string) time.Time {
	// Convert timestamp into unix timestamp
	// @todo make input format configurable
	lineTime, err := time.Parse("2006-01-02 15:04:05 -0700", inputDate)
	if err != nil {
		panic(err)
	}
	return lineTime
}

func (lr LogReader) IsUserIgnored(username string) bool {
	for _, n := range(lr.Config.Ignore) {
		if username == n {
			return true
		}
	}
	return false
}

// Map the users nick if found in the configuration against a mapping.
func (lr LogReader) MapNick(username string) string {
	if val, ok := lr.Config.NickNameHashTable[username]; ok{
		return val
	}
	return username
}

func (lr *LogReader) ParseLine(line string, isAction bool) bool {
	var parsed[][]string
	var user User

	// timestamp = [0][1]
	// nick = [0][2]
	// message/action = [0][3]

	if isAction == true {
		parsed = lr.RegexParseAction.FindAllStringSubmatch(line, -1)
	} else {
		parsed = lr.RegexParseMessage.FindAllStringSubmatch(line, -1)
	}

	// Convert timestamp into unix timestamp
	lineTime := lr.ParseTime(parsed[0][1])
	if lineTime.Unix() <= lr.Database.Channel.Last {
		return false
	}

	// Parse nick and check against ignore list
	lineNick := strings.Trim(strings.ToLower(parsed[0][2]), " ")
	if (lr.IsUserIgnored(lineNick) == true) {
		return false
	}
	lineNick = lr.MapNick(lineNick)

	// Parse message
	lineMessage := strings.Trim(parsed[0][3], " ")

	// If this is an empty line lets ignore it
	if lineMessage == "" {
		return false
	}

	// Get user, if not found make a new user
	if lr.Database.HasUser(lineNick) == true {
		user, _ = lr.Database.GetUser(lineNick)
	} else {
		user = NewUser(lineNick, lineTime.Unix())
	}

	lineMessageCharCount := (strings.Count(lineMessage, "") - 1)
	lineMessageWords := strings.Split(strings.ToLower(lineMessage), " ")
	lineMessageWordCount := len(lineMessageWords)

	user.Words = append(user.Words, lineMessageWords...)

	lr.Database.Channel.LineCount++
	user.LineCount++

	user.WordCount += uint(lineMessageWordCount)
	lr.Database.Channel.WordCount += uint(lineMessageWordCount)
	user.CharCount += uint(lineMessageCharCount)

	// Increment words per day
	user.IncrementDay(lineTime.Format("2006-02-01"), uint(lineMessageCharCount))
	lr.Database.IncrementDay(lineTime.Format("2006-02-01"), uint(lineMessageCharCount))

	// Increment lines per hour
	user.IncrementHour(uint(lineTime.Hour()))
	lr.Database.IncrementHour(uint(lineTime.Hour()))

	// Set first and last seen timestamps
	if user.FirstSeen > lineTime.Unix() {
		user.FirstSeen = lineTime.Unix()
	}

	if user.LastSeen < lineTime.Unix() {
		user.LastSeen = lineTime.Unix()
	}

	if lr.Database.Channel.First == 0 {
		lr.Database.Channel.First = lineTime.Unix()
	}

	if lr.Database.Channel.Last == 0 {
		lr.Database.Channel.Last = lineTime.Unix()
	}

	if lr.Database.Channel.First > lineTime.Unix() {
		lr.Database.Channel.First = lineTime.Unix()
	}<commit_msg>Fixed missunderstanding on difference between user days and database days<commit_after>package main

import (
	"fmt"
	"bufio"
	"regexp"
	"strings"
	"time"
	"os"
)

type LogReader struct {
	Database          Database
	RegexAction       *regexp.Regexp
	RegexMessage      *regexp.Regexp
	RegexParseAction  *regexp.Regexp
	RegexParseMessage *regexp.Regexp
	Config            Config
}

func (lr *LogReader) LoadFile() bool {
	// Open the file
	f, err := os.Open(lr.Config.Location)

	if (err != nil) {
		return false
	}

	// Create Buffered Scanner
	scanner := bufio.NewScanner(f)

	// Loop over all found lines and analyse them
	for scanner.Scan() {
		line := scanner.Text()
		if lr.RegexAction.MatchString(line) == true {
			lr.ParseLine(line, true)
		} else if lr.RegexMessage.MatchString(line) == true {
			lr.ParseLine(line, false)
		} else {
			fmt.Printf("error reading line [%d]\n", lr.Database.Channel.LineCount)
		}
	}

	lr.Database.LastGenerated = time.Now().Unix()

	return true;
}

func (lr LogReader) ParseTime(inputDate string) time.Time {
	// Convert timestamp into unix timestamp
	// @todo make input format configurable
	lineTime, err := time.Parse("2006-01-02 15:04:05 -0700", inputDate)
	if err != nil {
		panic(err)
	}
	return lineTime
}

func (lr LogReader) IsUserIgnored(username string) bool {
	for _, n := range(lr.Config.Ignore) {
		if username == n {
			return true
		}
	}
	return false
}

// Map the users nick if found in the configuration against a mapping.
func (lr LogReader) MapNick(username string) string {
	if val, ok := lr.Config.NickNameHashTable[username]; ok{
		return val
	}
	return username
}

func (lr *LogReader) ParseLine(line string, isAction bool) bool {
	var parsed[][]string
	var user User

	// timestamp = [0][1]
	// nick = [0][2]
	// message/action = [0][3]

	if isAction == true {
		parsed = lr.RegexParseAction.FindAllStringSubmatch(line, -1)
	} else {
		parsed = lr.RegexParseMessage.FindAllStringSubmatch(line, -1)
	}

	// Convert timestamp into unix timestamp
	lineTime := lr.ParseTime(parsed[0][1])
	if lineTime.Unix() <= lr.Database.Channel.Last {
		return false
	}

	// Parse nick and check against ignore list
	lineNick := strings.Trim(strings.ToLower(parsed[0][2]), " ")
	if (lr.IsUserIgnored(lineNick) == true) {
		return false
	}
	lineNick = lr.MapNick(lineNick)

	// Parse message
	lineMessage := strings.Trim(parsed[0][3], " ")

	// If this is an empty line lets ignore it
	if lineMessage == "" {
		return false
	}

	// Get user, if not found make a new user
	if lr.Database.HasUser(lineNick) == true {
		user, _ = lr.Database.GetUser(lineNick)
	} else {
		user = NewUser(lineNick, lineTime.Unix())
	}

	lineMessageCharCount := (strings.Count(lineMessage, "") - 1)
	lineMessageWords := strings.Split(strings.ToLower(lineMessage), " ")
	lineMessageWordCount := len(lineMessageWords)

	user.Words = append(user.Words, lineMessageWords...)

	lr.Database.Channel.LineCount++
	user.LineCount++

	user.WordCount += uint(lineMessageWordCount)
	lr.Database.Channel.WordCount += uint(lineMessageWordCount)
	user.CharCount += uint(lineMessageCharCount)

	// Increment words per day
	user.IncrementDay(lineTime.Format("2006-02-01"), uint(lineMessageCharCount))

	// Increment lines per day
	lr.Database.IncrementDay(lineTime.Format("2006-02-01"), 1)

	// Increment lines per hour
	user.IncrementHour(uint(lineTime.Hour()))
	lr.Database.IncrementHour(uint(lineTime.Hour()))

	// Set first and last seen timestamps
	if user.FirstSeen > lineTime.Unix() {
		user.FirstSeen = lineTime.Unix()
	}

	if user.LastSeen < lineTime.Unix() {
		user.LastSeen = lineTime.Unix()
	}

	if lr.Database.Channel.First == 0 {
		lr.Database.Channel.First = lineTime.Unix()
	}

	if lr.Database.Channel.Last == 0 {
		lr.Database.Channel.Last = lineTime.Unix()
	}

	if lr.Database.Channel.First > lineTime.Unix() {
		lr.Database.Channel.First = lineTime.Unix()
	}
{'fraction_non_alphanumeric': 0.1344058121432278, 'fraction_numerical': 0.013362740010378828, 'mean_word_length': 3.833228840125392, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2583463', 'n_tokens_mistral': 2869, 'n_tokens_neox': 2681, 'n_words': 621}