<commit_before>// BucketReader gets a list of files awaiting processing in the S3 intake buckets
// and adds each item to the queue for processing.
package main

import (
	"bytes"
	"flag"
	"log"
	"fmt"
	"encoding/json"
	"net/http"
	"github.com/APTrust/bagman"
)

var config bagman.Config
var jsonLog *log.Logger
var messageLog *log.Logger

func main() {
	initialize()
	run()
}

func initialize() {
	// Load the config or die.
	requestedConfig := flag.String("config", "", "configuration to run")
	flag.Parse()
	config = bagman.LoadRequestedConfig(requestedConfig)
	jsonLog, messageLog = bagman.InitLoggers(config.LogDirectory, "bucket_reader")
	bagman.PrintConfig(config)
}


func run() {
	bucketSummaries, err := bagman.CheckAllBuckets(config.Buckets)
	if err != nil {
		messageLog.Println("[ERROR]", err)
		return
	}
	url := fmt.Sprintf("%s/put?topic=%s", config.NsqdHttpAddress,
		config.BagProcessorTopic)
	messageLog.Printf("[INFO] Sending S3 file info to %s \n", url)
	for _, bucketSummary := range bucketSummaries {
		for _, key := range bucketSummary.Keys {
			if key.Size < config.MaxFileSize {
				// TODO: Set attempt number correctly when queue is working.
				s3File := bagman.S3File{bucketSummary.BucketName, key, nil}
				enqueue(url, s3File)
			}
		}
	}
}
<commit_msg>Minor changes to match other refactors<commit_after>// BucketReader gets a list of files awaiting processing in the S3 intake buckets
// and adds each item to the queue for processing.
package main

import (
	"bytes"
	"flag"
	"log"
	"fmt"
	"encoding/json"
	"net/http"
	"github.com/APTrust/bagman"
)

var config bagman.Config
var jsonLog *log.Logger
var messageLog *log.Logger

func main() {
	initialize()
	run()
}

func initialize() {
	// Load the config or die.
	requestedConfig := flag.String("config", "", "configuration to run")
	flag.Parse()
	config = bagman.LoadRequestedConfig(requestedConfig)
	jsonLog, messageLog = bagman.InitLoggers(config.LogDirectory, "bucket_reader")
}


func run() {
	bucketSummaries, err := bagman.CheckAllBuckets(config.Buckets)
	if err != nil {
		messageLog.Println("[ERROR]", err)
		return
	}
	url := fmt.Sprintf("%s/put?topic=%s", config.NsqdHttpAddress,
		config.BagProcessorTopic)
	messageLog.Printf("[INFO] Sending S3 file info to %s \n", url)
	for _, bucketSummary := range bucketSummaries {
		for _, key := range bucketSummary.Keys {
			if key.Size < config.MaxFileSize {
				s3File := bagman.S3File{bucketSummary.BucketName, key}
				enqueue(url, s3File)
			}
		}
	}
}

{'fraction_non_alphanumeric': 0.12585170340681362, 'fraction_numerical': 0.004008016032064128, 'mean_word_length': 4.2993630573248405, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '901576', 'n_tokens_mistral': 892, 'n_tokens_neox': 849, 'n_words': 226}