<commit_before>package workers

import (
	"fmt"
	"github.com/APTrust/easy-store/db/models"
	"io"
)

type JobRunner struct {
	Job       *models.Job
	LogWriter io.Writer
	Errors    []string
}

// TODO: track progress of bagging & upload

func NewJobRunner(job *models.Job, logWriter io.Writer) *JobRunner {
	return &JobRunner{
		Job:       job,
		LogWriter: logWriter,
	}
}

func (r JobRunner) Run() bool {
	if !r.ConfigDataOk() {
		return false
	}
	return true
}

func (r JobRunner) ConfigDataOk() bool {
	if r.Job == nil {
		r.AddError("Job cannot be null")
		return false
	}
	workflow, err := r.Job.Workflow()
	if workflow == nil {
		if err != nil {
			r.AddError("Cannot retrieve workflow: %v", err)
		} else {
			r.AddError("Cannot run job without workflow")
		}
		return false
	}
	// We'll get an error only if profile is set and is not
	// retrievable. There will be no error if the
	// workflow.ProfileId is nil. The same goes for the calls
	// to StorageService(), Bag(), and File().
	profile, err := workflow.Profile()
	if err != nil {
		r.AddError("Cannot retrieve BagIt profile for this workflow: %v", err)
		return false
	}
	storageService, err := workflow.StorageService()
	if err != nil {
		r.AddError("Cannot retrieve storage service for this workflow: %v", err)
		return false
	}
	bag, err := r.Job.Bag()
	if err != nil {
		r.AddError("Cannot retrieve the bag record for this job: %v", err)
		return false
	}
	// File will only exist if the Workflow does not include any
	// bagging. In that case, the Workflow is to copy the specified
	// file to the storage service.
	file, err := r.Job.File()
	if err != nil {
		r.AddError("Cannot retrieve the file record for this job: %v", err)
		return false
	}

	// The presence of a BagItProfile indicates that the job
	// includes creating a bag. The presence of a StorageService
	// indicates that the job includes copying a bag or file to
	// the remote storage service.
	if profile == nil && storageService == nil {
		r.AddError("This workflow has no BagIt profile and no storage service, so there " +
			"is nothing to do.")
		return false
	}
	if profile != nil && bag == nil {
		r.AddError("This workflow includes a bagging task, but the bag record is missing.")
		return false
	}
	if storageService != nil && bag == nil && file == nil {
		r.AddError("This workflow includes an upload task, but no bag or file is specified.")
		return false
	}

	// TODO: if bag is specified, make sure it has files.

	return true
}

func (r JobRunner) AddError(format string, a ...interface{}) {
	msg := fmt.Sprintf(format, a...)
	r.Errors = append(r.Errors, msg)
	r.Log(format, a...)
}

func (r JobRunner) Log(format string, a ...interface{}) {
	if r.LogWriter != nil {
		msg := fmt.Sprintf(format, a...) + "\n"
		r.LogWriter.Write([]byte(msg))
	}
}<commit_msg>Working on JobRunner<commit_after>package workers

import (
	"fmt"
	"github.com/APTrust/easy-store/db/models"
	"io"
)

type JobRunner struct {
	Job            *models.Job
	LogWriter      io.Writer
	Errors         []string
	workflow       *models.Workflow
	profile        *models.BagItProfile
	storageService *models.StorageService
	bag            *models.Bag
	file           *models.File
	files          []*models.File
}

// TODO: track progress of bagging & upload

func NewJobRunner(job *models.Job, logWriter io.Writer) *JobRunner {
	return &JobRunner{
		Job:       job,
		LogWriter: logWriter,
	}
}

func (r JobRunner) Run() bool {
	if !r.ConfigDataOk() {
		return false
	}
	return true
}

// BuildBag assembles JobRunner.Bag.Files() into a bag using the
// BagItProfile specified in JobRunner.Job.Profile(). If there's
// no bag or profile specified, this is a no-op.
func (r JobRunner) BuildBag() bool {
	if r.profile && r.bag {

	}
	return true
}

// SerializeBag serializes the bag to the specified format.
// Currently, tar is the only supported format.
func (r JobRunner) SerializeBag() bool {
	if r.workflow.SerializationFormat == "tar" {

	}
	return true
}

// CopyBagToRemote copies the bag to the remote storage area.
// For now, only S3 is supported. In future, we may add SFTP
// and other protocols.
func (r JobRunner) CopyBagToRemote() bool {
	if r.storageService != nil {

	}
	return true
}

func (r JobRunner) ConfigDataOk() bool {
	if r.Job == nil {
		r.AddError("Job cannot be null")
		return false
	}
	var err error
	r.workflow, err = r.Job.Workflow()
	if r.workflow == nil {
		if err != nil {
			r.AddError("Cannot retrieve workflow: %v", err)
		} else {
			r.AddError("Cannot run job without workflow")
		}
		return false
	}
	// We'll get an error only if profile is set and is not
	// retrievable. There will be no error if the
	// workflow.ProfileId is nil. The same goes for the calls
	// to StorageService(), Bag(), and File().
	r.profile, err = r.workflow.Profile()
	if err != nil {
		r.AddError("Cannot retrieve BagIt profile for this workflow: %v", err)
		return false
	}
	r.storageService, err = r.workflow.StorageService()
	if err != nil {
		r.AddError("Cannot retrieve storage service for this workflow: %v", err)
		return false
	}
	r.bag, err = r.Job.Bag()
	if err != nil {
		r.AddError("Cannot retrieve the bag record for this job: %v", err)
		return false
	}
	// File will only exist if the Workflow does not include any
	// bagging. In that case, the Workflow is to copy the specified
	// file to the storage service.
	r.file, err = r.Job.File()
	if err != nil {
		r.AddError("Cannot retrieve the file record for this job: %v", err)
		return false
	}

	// The presence of a BagItProfile indicates that the job
	// includes creating a bag. The presence of a StorageService
	// indicates that the job includes copying a bag or file to
	// the remote storage service.
	if r.profile == nil && r.storageService == nil {
		r.AddError("This workflow has no BagIt profile and no storage service, so there " +
			"is nothing to do.")
		return false
	}
	if r.profile != nil && r.bag == nil {
		r.AddError("This workflow includes a bagging task, but the bag record is missing.")
		return false
	}
	if r.storageService != nil && r.bag == nil && r.file == nil {
		r.AddError("This workflow includes an upload task, but no bag or file is specified.")
		return false
	}
	if r.bag != nil {
		r.files, err = r.bag.Files()
		if err != nil {
			r.AddError("Cannot get files for bag %s: %s", r.bag.Name, err.Error())
			return false
		} else if len(r.files) == 0 {
			r.AddError("Bag %s has no files.", r.bag.Name)
			return false
		}
	}
	return true
}

func (r JobRunner) AddError(format string, a ...interface{}) {
	msg := fmt.Sprintf(format, a...)
	r.Errors = append(r.Errors, msg)
	r.Log(format, a...)
}

func (r JobRunner) Log(format string, a ...interface{}) {
	if r.LogWriter != nil {
		msg := fmt.Sprintf(format, a...) + "\n"
		r.LogWriter.Write([]byte(msg))
	}
}
{'fraction_non_alphanumeric': 0.12224183583406885, 'fraction_numerical': 0.0002942041776993233, 'mean_word_length': 3.1131276467029645, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1165892', 'n_tokens_mistral': 2312, 'n_tokens_neox': 2199, 'n_words': 833}