<commit_before>package mdbd

import (
	"encoding/gob"
	"encoding/json"
	"github.com/Symantec/Dominator/lib/mdb"
	"log"
	"os"
	"path"
	"sort"
	"syscall"
	"time"
)

func startMdbDaemon(mdbFileName string, logger *log.Logger) <-chan *mdb.Mdb {
	mdbChannel := make(chan *mdb.Mdb)
	go watchDaemon(mdbFileName, mdbChannel, logger)
	return mdbChannel
}

type genericDecoder interface {
	Decode(v interface{}) error
}

func watchDaemon(mdbFileName string, mdbChannel chan<- *mdb.Mdb,
	logger *log.Logger) {
	var lastStat syscall.Stat_t
	var lastMdb *mdb.Mdb
	for ; ; time.Sleep(time.Second) {
		var stat syscall.Stat_t
		if err := syscall.Stat(mdbFileName, &stat); err != nil {
			logger.Printf("Error stating file: %s\t%s\n", mdbFileName, err)
			continue
		}
		stat.Atim = lastStat.Atim
		if stat != lastStat {
			mdb := loadFile(mdbFileName, logger)
			if mdb == nil {
				continue
			}
			compareStartTime := time.Now()
			if lastMdb == nil || !compare(lastMdb, mdb) {
				if lastMdb != nil {
					mdbCompareTimeDistribution.Add(time.Since(compareStartTime))
				}
				mdbChannel <- mdb
				lastMdb = mdb
			}
			lastStat = stat
		}
	}
}

func loadFile(mdbFileName string, logger *log.Logger) *mdb.Mdb {
	file, err := os.Open(mdbFileName)
	if err != nil {
		logger.Printf("Error opening file\t%s\n", err)
		return nil
	}
	decoder := getDecoder(file)
	var mdb mdb.Mdb
	decodeStartTime := time.Now()
	if err = decoder.Decode(&mdb.Machines); err != nil {
		logger.Printf("Error decoding\t%s\n", err)
		return nil
	}
	sortStartTime := time.Now()
	mdbDecodeTimeDistribution.Add(sortStartTime.Sub(decodeStartTime))
	sort.Sort(&mdb)
	mdbSortTimeDistribution.Add(time.Since(sortStartTime))
	return &mdb<commit_msg>Close MDB file after use in dominator.<commit_after>package mdbd

import (
	"encoding/gob"
	"encoding/json"
	"github.com/Symantec/Dominator/lib/mdb"
	"log"
	"os"
	"path"
	"sort"
	"syscall"
	"time"
)

func startMdbDaemon(mdbFileName string, logger *log.Logger) <-chan *mdb.Mdb {
	mdbChannel := make(chan *mdb.Mdb)
	go watchDaemon(mdbFileName, mdbChannel, logger)
	return mdbChannel
}

type genericDecoder interface {
	Decode(v interface{}) error
}

func watchDaemon(mdbFileName string, mdbChannel chan<- *mdb.Mdb,
	logger *log.Logger) {
	var lastStat syscall.Stat_t
	var lastMdb *mdb.Mdb
	for ; ; time.Sleep(time.Second) {
		var stat syscall.Stat_t
		if err := syscall.Stat(mdbFileName, &stat); err != nil {
			logger.Printf("Error stating file: %s\t%s\n", mdbFileName, err)
			continue
		}
		stat.Atim = lastStat.Atim
		if stat != lastStat {
			mdb := loadFile(mdbFileName, logger)
			if mdb == nil {
				continue
			}
			compareStartTime := time.Now()
			if lastMdb == nil || !compare(lastMdb, mdb) {
				if lastMdb != nil {
					mdbCompareTimeDistribution.Add(time.Since(compareStartTime))
				}
				mdbChannel <- mdb
				lastMdb = mdb
			}
			lastStat = stat
		}
	}
}

func loadFile(mdbFileName string, logger *log.Logger) *mdb.Mdb {
	file, err := os.Open(mdbFileName)
	if err != nil {
		logger.Printf("Error opening file\t%s\n", err)
		return nil
	}
	defer file.Close()
	decoder := getDecoder(file)
	var mdb mdb.Mdb
	decodeStartTime := time.Now()
	if err = decoder.Decode(&mdb.Machines); err != nil {
		logger.Printf("Error decoding\t%s\n", err)
		return nil
	}
	sortStartTime := time.Now()
	mdbDecodeTimeDistribution.Add(sortStartTime.Sub(decodeStartTime))
	sort.Sort(&mdb)
	mdbSortTimeDistribution.Add(time.Since(sortStartTime))
	return &mdb
{'fraction_non_alphanumeric': 0.13395187010727747, 'fraction_numerical': 0.0, 'mean_word_length': 3.985549132947977, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3579132', 'n_tokens_mistral': 1438, 'n_tokens_neox': 1311, 'n_words': 281}