<commit_before>package core

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"strconv"

	"github.com/robfig/cron"
	"github.com/sajari/regression"
	log "github.com/sirupsen/logrus"
	"github.com/vkuznet/transfer2go/utils"
)

type Router struct {
	CronInterval     string                 // Helps to set hourly based cron job
	LinearRegression *regression.Regression // machine learning model
	CSVfile          string                 // historical data file
	Agents           *map[string]string     // list of connected agents
}

// AgentRouter helps to call router's methods
var AgentRouter Router

// newRouter returns new instance of Router type
func NewRouter(interval string, agent *map[string]string, csvFile string) *cron.Cron {
	lr := new(regression.Regression)
	lr.SetObserved("Get throughput")
	lr.SetVar(0, "CPU usage")
	lr.SetVar(1, "Memory usage")

	timeConfig := "@every " + interval // It works on this format - http://golang.org/pkg/time/#ParseDuration
	c := cron.New()
	c.AddFunc(timeConfig, train)
	AgentRouter = Router{CronInterval: interval, Agents: agent, LinearRegression: lr, CSVfile: csvFile}
	return c
}

// Function to train the agent
func train() {
	var dataPoints []TransferData
	for _, source := range *AgentRouter.Agents {
		data, err := getHistory(source)
		if err == nil {
			dataPoints = append(dataPoints, data...)
		}
	}
	if len(dataPoints) == 0 {
		return
	}

	// Reinitialize the model
	lr := new(regression.Regression)
	lr.SetObserved("Get throughput")
	lr.SetVar(0, "CPU usage")
	lr.SetVar(1, "Memory usage")
	AgentRouter.LinearRegression = lr

	for _, obj := range dataPoints {
		AgentRouter.LinearRegression.Train(regression.DataPoint(obj.Throughput, []float64{obj.CpuUsage, obj.MemUsage}))
	}
	AgentRouter.LinearRegression.Run()
	log.WithFields(log.Fields{
		"CronInterval": AgentRouter.CronInterval,
		"Data":         AgentRouter.CSVfile,
	}).Println("Router successfully retrained")

	err := convertToCSV(dataPoints)
	if err != nil {
		log.WithFields(log.Fields{
			"Error": err,
		}).Println("Error while saving data")
	}
}

// Convert struct to csv format
func convertToCSV(dataPoints []TransferData) error {
	file, err := os.Create(AgentRouter.CSVfile)
	if err != nil {
		return err
	}
	defer file.Close()

	writer := csv.NewWriter(file)
	defer writer.Flush()

	for _, obj := range dataPoints {
		str := []string{strconv.FormatFloat(obj.CpuUsage, 'f', -1, 64), strconv.FormatFloat(obj.MemUsage, 'f', -1, 64), strconv.FormatFloat(obj.Throughput, 'f', -1, 64)}
		writer.Write(str)
	}
	return nil
}

// function to get historical data of agent
func getHistory(source string) ([]TransferData, error) {
	url := fmt.Sprintf("%s/history?duration=%s", source, "24h")
	resp := utils.FetchResponse(url, []byte{})
	if resp.Error != nil {
		return nil, resp.Error
	}
	var transferRecords []TransferData
	err := json.Unmarshal(resp.Data, &transferRecords)
	if err != nil {<commit_msg>Change parameter<commit_after>package core

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"strconv"

	"github.com/robfig/cron"
	"github.com/sajari/regression"
	log "github.com/sirupsen/logrus"
	"github.com/vkuznet/transfer2go/utils"
)

type Router struct {
	CronInterval     string                 // Helps to set hourly based cron job
	LinearRegression *regression.Regression // machine learning model
	CSVfile          string                 // historical data file
	Agents           *map[string]string     // list of connected agents
}

// AgentRouter helps to call router's methods
var AgentRouter Router

// newRouter returns new instance of Router type
func NewRouter(interval string, agent *map[string]string, csvFile string) *cron.Cron {
	lr := new(regression.Regression)
	lr.SetObserved("Get throughput")
	lr.SetVar(0, "CPU usage")
	lr.SetVar(1, "Memory usage")

	timeConfig := "@every " + interval // It works on this format - http://golang.org/pkg/time/#ParseDuration
	c := cron.New()
	c.AddFunc(timeConfig, train)
	AgentRouter = Router{CronInterval: interval, Agents: agent, LinearRegression: lr, CSVfile: csvFile}
	return c
}

// Function to train the agent
func train() {
	var dataPoints []TransferData
	for _, source := range *AgentRouter.Agents {
		data, err := getHistory(source)
		if err == nil {
			dataPoints = append(dataPoints, data...)
		}
	}
	if len(dataPoints) == 0 {
		return
	}

	// Reinitialize the model
	lr := new(regression.Regression)
	lr.SetObserved("Get throughput")
	lr.SetVar(0, "CPU usage")
	lr.SetVar(1, "Memory usage")
	AgentRouter.LinearRegression = lr

	for _, obj := range dataPoints {
		AgentRouter.LinearRegression.Train(regression.DataPoint(obj.Throughput, []float64{obj.CpuUsage, obj.MemUsage}))
	}
	AgentRouter.LinearRegression.Run()
	log.WithFields(log.Fields{
		"CronInterval": AgentRouter.CronInterval,
		"Data":         AgentRouter.CSVfile,
	}).Println("Router successfully retrained")

	err := convertToCSV(dataPoints)
	if err != nil {
		log.WithFields(log.Fields{
			"Error": err,
		}).Println("Error while saving data")
	}
}

// Convert struct to csv format
func convertToCSV(dataPoints []TransferData) error {
	file, err := os.Create(AgentRouter.CSVfile)
	if err != nil {
		return err
	}
	defer file.Close()

	writer := csv.NewWriter(file)
	defer writer.Flush()

	for _, obj := range dataPoints {
		str := []string{strconv.FormatFloat(obj.CpuUsage, 'f', -1, 64), strconv.FormatFloat(obj.MemUsage, 'f', -1, 64), strconv.FormatFloat(obj.Throughput, 'f', -1, 64)}
		writer.Write(str)
	}
	return nil
}

// function to get historical data of agent
func getHistory(source string) ([]TransferData, error) {
	url := fmt.Sprintf("%s/history?duration=%s", source, AgentRouter.CronInterval)
	resp := utils.FetchResponse(url, []byte{})
	if resp.Error != nil {
		return nil, resp.Error
	}
	var transferRecords []TransferData
	err := json.Unmarshal(resp.Data, &transferRecords)
	if err != nil {
{'fraction_non_alphanumeric': 0.1339979722879351, 'fraction_numerical': 0.006083136194660358, 'mean_word_length': 4.011854360711261, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3656580', 'n_tokens_mistral': 2082, 'n_tokens_neox': 1978, 'n_words': 499}