<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/MG-RAST/AWE/conf"
	. "github.com/MG-RAST/AWE/core"
	. "github.com/MG-RAST/AWE/logger"
	"io/ioutil"
	"os"
	"os/exec"
	"time"
)

func deliverer(control chan int) {
	fmt.Printf("deliverer lanched, client=%s\n", self.Id)
	defer fmt.Printf("deliverer exiting...\n")
	for {
		processed := <-chanProcessed
		work := processed.workunit
		perfstat := processed.perfstat

		//post-process for works computed successfully: push output data to Shock
		move_start := time.Now().Unix()
		if processed.status == WORK_STAT_COMPUTED {
			if err := pushOutputData(work); err != nil {
				processed.status = WORK_STAT_FAIL
			} else {
				processed.status = WORK_STAT_DONE
			}
		}
		move_end := time.Now().Unix()
		perfstat.DataOut = move_end - move_start
		perfstat.Deliver = move_end
		perfstat.ClientResp = perfstat.Deliver - perfstat.Checkout
		perfstat.ClientId = self.Id

		//notify server the final process results
		if err := notifyWorkunitProcessed(work, processed.status, perfstat); err != nil {
			time.Sleep(3 * time.Second) //wait 3 seconds and try another time
			if err := notifyWorkunitProcessed(work, processed.status, perfstat); err != nil {
				fmt.Printf("!!!NotifyWorkunitDone returned error: %s\n", err.Error())
				Log.Error("err@NotifyWorkunitProcessed: workid=" + work.Id + ", err=" + err.Error())
				//mark this work in Current_work map as false, something needs to be done in the future
				//to clean this kind of work that has been proccessed but its result can't be sent to server!
				self.Current_work[work.Id] = false //server doesn't know this yet<commit_msg>more error log msg<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/MG-RAST/AWE/conf"
	. "github.com/MG-RAST/AWE/core"
	. "github.com/MG-RAST/AWE/logger"
	"io/ioutil"
	"os"
	"os/exec"
	"time"
)

func deliverer(control chan int) {
	fmt.Printf("deliverer lanched, client=%s\n", self.Id)
	defer fmt.Printf("deliverer exiting...\n")
	for {
		processed := <-chanProcessed
		work := processed.workunit
		perfstat := processed.perfstat

		//post-process for works computed successfully: push output data to Shock
		move_start := time.Now().Unix()
		if processed.status == WORK_STAT_COMPUTED {
			if err := pushOutputData(work); err != nil {
				processed.status = WORK_STAT_FAIL
				Log.Error("err@pushOutputData: workid=" + work.Id + ", err=" + err.Error())
			} else {
				processed.status = WORK_STAT_DONE
			}
		}
		move_end := time.Now().Unix()
		perfstat.DataOut = move_end - move_start
		perfstat.Deliver = move_end
		perfstat.ClientResp = perfstat.Deliver - perfstat.Checkout
		perfstat.ClientId = self.Id

		//notify server the final process results
		if err := notifyWorkunitProcessed(work, processed.status, perfstat); err != nil {
			time.Sleep(3 * time.Second) //wait 3 seconds and try another time
			if err := notifyWorkunitProcessed(work, processed.status, perfstat); err != nil {
				fmt.Printf("!!!NotifyWorkunitDone returned error: %s\n", err.Error())
				Log.Error("err@NotifyWorkunitProcessed: workid=" + work.Id + ", err=" + err.Error())
				//mark this work in Current_work map as false, something needs to be done in the future
				//to clean this kind of work that has been proccessed but its result can't be sent to server!
				self.Current_work[work.Id] = false //server doesn't know this yet
{'fraction_non_alphanumeric': 0.1300264162019372, 'fraction_numerical': 0.001174053419430584, 'mean_word_length': 3.773109243697479, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5864648', 'n_tokens_mistral': 1257, 'n_tokens_neox': 1137, 'n_words': 303}