<commit_before>// Package job provides the Job type and some methods on it.
// TODO probably need a document type
package job

import(
  "launchpad.net/goamz/aws"
  "launchpad.net/goamz/s3"
  "log"
  "fmt"
  "strings"
  "encoding/json"
)

// A Job includes all the data necessary to execute a pdf combination.
// It is mainly constructed from a JSON string in a HTTP request, but the
// last two fields contain internal state.
type Job struct {
  BucketName string     `json:"bucket_name"`
  EmployerId int        `json:"employer_id"`
  DocList    []string   `json:"doc_list"`
  Downloaded []string   `json:"downloaded"`
  Callback   string     `json:"callback"`
  Errors     []error    `json:"errors"`
  bucket     *s3.Bucket
}

// Does the job contain all the fields necessary to start the combination?
func (j *Job) IsValid() bool {
 return (j.BucketName != "") &&
        (j.Callback   != "") &&
        (j.EmployerId > 0)   &&
        (j.DocCount() > 0)
}

// Retrieve the requested document from S3 as a byte slice.
func (j *Job) Get(docname string) (data []byte, err error) {
  //TODO j.connect() if necessary
  data, err = j.bucket.Get(j.s3Path(docname))
  return
}

// The number of documents requested.
func (j *Job) DocCount() int {
  return len(j.DocList)
}

// The number of documents actually completed.
func (j *Job) CompleteCount() int {
  return len(j.Downloaded)
}

// Add a document to the list of completed (downloaded) docs.
func (j *Job) MarkComplete(newdoc string) {
  j.Downloaded = append(j.Downloaded, newdoc)
}

// Have any documents been successfully downloaded?
func (j *Job) HasDownloadedDocs() bool {
  return len(j.Downloaded) > 0
}

func (j *Job) RecipientUrl() string{
  return j.Callback
}

func (j *Job) ToJson() (response []byte) {
  response, _ = json.Marshal(j)
  fmt.Printf("%s\n",response)
  return
}

// Add to the list of encountered errors, translating obscure ones.
func (j *Job) AddError(newErr error) {
  log.Println(newErr)
  if strings.Contains(newErr.Error(), "Get : 301 response missing Location header") {
    newErr = fmt.Errorf("bucket %s not accessible from this account", j.BucketName)
  }
  j.Errors = append(j.Errors, newErr)
}

// Connect to AWS and add the handle to the Job object.
func (j *Job) Connect() {
  auth, err := aws.EnvAuth()
  if err != nil { panic(err) }<commit_msg>Push the job down to a second level object<commit_after>// Package job provides the Job type and some methods on it.
// TODO probably need a document type
package job

import(
  "launchpad.net/goamz/aws"
  "launchpad.net/goamz/s3"
  "log"
  "fmt"
  "strings"
  "encoding/json"
)

// A Job includes all the data necessary to execute a pdf combination.
// It is mainly constructed from a JSON string in a HTTP request, but the
// last two fields contain internal state.
type Job struct {
  BucketName string     `json:"bucket_name"`
  EmployerId int        `json:"employer_id"`
  DocList    []string   `json:"doc_list"`
  Downloaded []string   `json:"downloaded"`
  Callback   string     `json:"callback"`
  Errors     []error    `json:"errors"`
  bucket     *s3.Bucket
}

type JobResponse struct {
  Success bool `json:"success"`
  Job     Job  `json:"job"`
}

// Does the job contain all the fields necessary to start the combination?
func (j *Job) IsValid() bool {
 return (j.BucketName != "") &&
        (j.Callback   != "") &&
        (j.EmployerId > 0)   &&
        (j.DocCount() > 0)
}

// Retrieve the requested document from S3 as a byte slice.
func (j *Job) Get(docname string) (data []byte, err error) {
  //TODO j.connect() if necessary
  data, err = j.bucket.Get(j.s3Path(docname))
  return
}

// The number of documents requested.
func (j *Job) DocCount() int {
  return len(j.DocList)
}

// The number of documents actually completed.
func (j *Job) CompleteCount() int {
  return len(j.Downloaded)
}

// Add a document to the list of completed (downloaded) docs.
func (j *Job) MarkComplete(newdoc string) {
  j.Downloaded = append(j.Downloaded, newdoc)
}

// Have any documents been successfully downloaded?
// TODO is it appropriate to use this to determine success in ToJSON()?
func (j *Job) HasDownloadedDocs() bool {
  return len(j.Downloaded) > 0
}

func (j *Job) RecipientUrl() string{
  return j.Callback
}

func (j *Job) ToJson() (json_response []byte) {
  response := JobResponse{
    Job: *j,
    Success: j.HasDownloadedDocs() }
  json_response, _ = json.Marshal(response)
  fmt.Printf("%s\n",json_response)
  return
}

// Add to the list of encountered errors, translating obscure ones.
func (j *Job) AddError(newErr error) {
  log.Println(newErr)
  if strings.Contains(newErr.Error(), "Get : 301 response missing Location header") {
    newErr = fmt.Errorf("bucket %s not accessible from this account", j.BucketName)
  }
  j.Errors = append(j.Errors, newErr)
}

// Connect to AWS and add the handle to the Job object.
func (j *Job) Connect() {
  auth, err := aws.EnvAuth()
  if err != nil { panic(err) }
{'fraction_non_alphanumeric': 0.1259810827128195, 'fraction_numerical': 0.004024954719259408, 'mean_word_length': 3.421708185053381, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '282211', 'n_tokens_mistral': 1632, 'n_tokens_neox': 1595, 'n_words': 565}