<commit_before>package worker

import (
	"fmt"
	"github.com/MG-RAST/AWE/lib/conf"
	"github.com/MG-RAST/AWE/lib/core"
	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/MG-RAST/AWE/lib/logger/event"
	"os"
	"time"
)

func deliverer(control chan int) {
	fmt.Printf("deliverer lanched, client=%s\n", core.Self.Id)
	defer fmt.Printf("deliverer exiting...\n")
	for {
		processed := <-fromProcessor
		work := processed.workunit
		workmap[work.Id] = ID_DELIVERER
		perfstat := processed.perfstat

		//post-process for works computed successfully: push output data to Shock
		move_start := time.Now().UnixNano()
		if work.State == core.WORK_STAT_COMPUTED {
			if data_moved, err := core.PushOutputData(work); err != nil {
				work.State = core.WORK_STAT_FAIL
				logger.Error("err@pushOutputData: workid=" + work.Id + ", err=" + err.Error())
			} else {
				work.State = core.WORK_STAT_DONE
				perfstat.OutFileSize = data_moved
			}
		}
		move_end := time.Now().UnixNano()
		perfstat.DataOut = float64(move_end-move_start) / 1e9
		perfstat.Deliver = move_end
		perfstat.ClientResp = perfstat.Deliver - perfstat.Checkout
		perfstat.ClientId = core.Self.Id

		//notify server the final process results
		/*		if err := core.NotifyWorkunitProcessed(work, perfstat); err != nil {
					time.Sleep(3 * time.Second) //wait 3 seconds and try another time
					if err := core.NotifyWorkunitProcessed(work, perfstat); err != nil {
						fmt.Printf("!!!NotifyWorkunitDone returned error: %s\n", err.Error())
						logger.Error("err@NotifyWorkunitProcessed: workid=" + work.Id + ", err=" + err.Error())
						//mark this work in Current_work map as false, something needs to be done in the future
						//to clean this kind of work that has been proccessed but its result can't be sent to server!
						core.Self.Current_work[work.Id] = false //server doesn't know this yet
					}
				}
		*/<commit_msg>fix workperf deliver time nanosec -> sec<commit_after>package worker

import (
	"fmt"
	"github.com/MG-RAST/AWE/lib/conf"
	"github.com/MG-RAST/AWE/lib/core"
	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/MG-RAST/AWE/lib/logger/event"
	"os"
	"time"
)

func deliverer(control chan int) {
	fmt.Printf("deliverer lanched, client=%s\n", core.Self.Id)
	defer fmt.Printf("deliverer exiting...\n")
	for {
		processed := <-fromProcessor
		work := processed.workunit
		workmap[work.Id] = ID_DELIVERER
		perfstat := processed.perfstat

		//post-process for works computed successfully: push output data to Shock
		move_start := time.Now().UnixNano()
		if work.State == core.WORK_STAT_COMPUTED {
			if data_moved, err := core.PushOutputData(work); err != nil {
				work.State = core.WORK_STAT_FAIL
				logger.Error("err@pushOutputData: workid=" + work.Id + ", err=" + err.Error())
			} else {
				work.State = core.WORK_STAT_DONE
				perfstat.OutFileSize = data_moved
			}
		}
		move_end := time.Now().UnixNano()
		perfstat.DataOut = float64(move_end-move_start) / 1e9
		perfstat.Deliver = int64(move_end / 1e9)
		perfstat.ClientResp = perfstat.Deliver - perfstat.Checkout
		perfstat.ClientId = core.Self.Id

		//notify server the final process results
		/*		if err := core.NotifyWorkunitProcessed(work, perfstat); err != nil {
					time.Sleep(3 * time.Second) //wait 3 seconds and try another time
					if err := core.NotifyWorkunitProcessed(work, perfstat); err != nil {
						fmt.Printf("!!!NotifyWorkunitDone returned error: %s\n", err.Error())
						logger.Error("err@NotifyWorkunitProcessed: workid=" + work.Id + ", err=" + err.Error())
						//mark this work in Current_work map as false, something needs to be done in the future
						//to clean this kind of work that has been proccessed but its result can't be sent to server!
						core.Self.Current_work[work.Id] = false //server doesn't know this yet
					}
				}
		*/
{'fraction_non_alphanumeric': 0.13335086796422935, 'fraction_numerical': 0.004208311415044713, 'mean_word_length': 3.5436081242532858, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1424513', 'n_tokens_mistral': 1478, 'n_tokens_neox': 1305, 'n_words': 312}