<commit_before>package lib

import (
	"strings"

	"github.com/snickers/snickers/db"
	"github.com/snickers/snickers/types"
)

// DownloadFunc is a function type for the multiple
// possible ways to download the source file
type DownloadFunc func(jobID string) error

// StartJob starts the job
func StartJob(job types.Job) {
	dbInstance, _ := db.GetDatabase()

	downloadFunc := GetDownloadFunc(job.Source)
	if err := downloadFunc(job.ID); err != nil {
		job.Status = types.JobError
		job.Details = err.Error()
		dbInstance.UpdateJob(job.ID, job)
		return
	}

	if err := FFMPEGEncode(job.ID); err != nil {
		job.Status = types.JobError
		job.Details = err.Error()
		dbInstance.UpdateJob(job.ID, job)
		return
	}

	if err := S3Upload(job.ID); err != nil {
		job.Status = types.JobError
		job.Details = err.Error()
		dbInstance.UpdateJob(job.ID, job)
		return
	}

	job.Status = types.JobFinished
	dbInstance.UpdateJob(job.ID, job)
}

// GetDownloadFunc returns the download function
// based on the job source.
func GetDownloadFunc(jobSource string) DownloadFunc {
	if strings.Contains(jobSource, "amazonaws") {
		return S3Download
	}

	return HTTPDownload
}<commit_msg>log the job steps<commit_after>package lib

import (
	"strings"

	"github.com/snickers/snickers/db"
	"github.com/snickers/snickers/types"

	"github.com/apex/log"
	"github.com/apex/log/handlers/text"
)

// DownloadFunc is a function type for the multiple
// possible ways to download the source file
type DownloadFunc func(jobID string) error

// StartJob starts the job
func StartJob(job types.Job) {
	dbInstance, _ := db.GetDatabase()

	log.SetHandler(text.New(GetLogOutput()))
	ctx := log.WithFields(log.Fields{
		"id":          job.ID,
		"status":      job.Status,
		"source":      job.Source,
		"destination": job.Destination,
	})

	ctx.Info("Downloading")
	downloadFunc := GetDownloadFunc(job.Source)
	if err := downloadFunc(job.ID); err != nil {
		ctx.WithError(err).Error("Download Failed")
		job.Status = types.JobError
		job.Details = err.Error()
		dbInstance.UpdateJob(job.ID, job)
		return
	}

	ctx.Info("Encodinging")
	if err := FFMPEGEncode(job.ID); err != nil {
		ctx.WithError(err).Error("Encode Failed")
		job.Status = types.JobError
		job.Details = err.Error()
		dbInstance.UpdateJob(job.ID, job)
		return
	}

	ctx.Info("Uploading")
	if err := S3Upload(job.ID); err != nil {
		ctx.WithError(err).Error("Upload Failed")
		job.Status = types.JobError
		job.Details = err.Error()
		dbInstance.UpdateJob(job.ID, job)
		return
	}

	job.Status = types.JobFinished
	dbInstance.UpdateJob(job.ID, job)
}

// GetDownloadFunc returns the download function
// based on the job source.
func GetDownloadFunc(jobSource string) DownloadFunc {
	if strings.Contains(jobSource, "amazonaws") {
		return S3Download
	}

	return HTTPDownload
}
{'fraction_non_alphanumeric': 0.12879058151980022, 'fraction_numerical': 0.0014270424545130217, 'mean_word_length': 4.007142857142857, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1136420', 'n_tokens_mistral': 1002, 'n_tokens_neox': 951, 'n_words': 221}