<commit_before>package main

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log"
	"math/rand"
	"sync"

	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/sqs"
)

//QuestionQ : name of the questions SQS Queue
const QuestionQ = "demo-questions"

//AnswerQ : name of the answers SQS Queue
const AnswerQ = "demo-answers"

//SQS instace of the sqs var from goamz
var SQS *sqs.SQS

var wg sync.WaitGroup

//Question : struct that holds the data for the math problem.
type Question struct {
	Num1 int
	Num2 int
}

func main() {

}

func makeRun(public string, secret string, maxworkers int) error {
	auth := aws.Auth{AccessKey: public, SecretKey: secret}
	region := aws.Region{}
	region.Name = "us-west-2"
	region.SQSEndpoint = "http://sqs.us-west-2.amazonaws.com"
	SQS = sqs.New(auth, region)
	if SQS == nil {
		return fmt.Errorf("Can't get sqs reference for %v %v", auth, region)
	}
	questionq, getErr := SQS.GetQueue(QuestionQ)
	if getErr != nil {
		fmt.Println(QuestionQ)
		return getErr
	}
	log.Println("Start")
	for i := 0; i < 250; i++ {
		var msgList []sqs.Message
		for c := 0; c < 10; c++ {
			num1 := rand.Intn(9) + 1
			num2 := rand.Intn(9 + 1)
			q := Question{num1, num2}
			jsonQ, _ := json.Marshal(&q)
			msg := sqs.Message{}
			body := base64.StdEncoding.EncodeToString(jsonQ)
			msg.Body = body
			msgList = append(msgList, msg)
		}
		wg.Add(1)
		go addToQuestionQ(msgList, *questionq)
	}
	wg.Wait()
	log.Println("End")
	return nil
}

func addToQuestionQ(msgList []sqs.Message, sqsQ sqs.Queue) {
	_, respErr := sqsQ.SendMessageBatch(msgList)
	if respErr != nil {
		log.Println("ERROR:", respErr)
	}
	wg.Done()
}<commit_msg>trying to implement a buffered channel<commit_after>package main

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log"
	"math/rand"
	"sync"

	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/sqs"
)

//QuestionQ : name of the questions SQS Queue
const QuestionQ = "demo-questions"

//AnswerQ : name of the answers SQS Queue
const AnswerQ = "demo-answers"

//SQS instace of the sqs var from goamz
var SQS *sqs.SQS

var wg sync.WaitGroup

//Question : struct that holds the data for the math problem.
type Question struct {
	Num1 int
	Num2 int
}

var sem = make(chan int, 250)

func main() {

}

func Serve(queue chan []sqs.Message, sqsQ sqs.Queue) {
	for q := range queue {
		sem <- 1
		go func() {
			q := q
			sem <- 1
			go func() {
				addToQuestionQ(q, sqsQ)
				<-sem
			}()
		}()
	}
}

func makeRun(public string, secret string, maxworkers int) error {

	auth := aws.Auth{AccessKey: public, SecretKey: secret}
	region := aws.Region{}
	region.Name = "us-west-2"
	region.SQSEndpoint = "http://sqs.us-west-2.amazonaws.com"
	SQS = sqs.New(auth, region)
	if SQS == nil {
		return fmt.Errorf("Can't get sqs reference for %v %v", auth, region)
	}
	questionq, getErr := SQS.GetQueue(QuestionQ)
	if getErr != nil {
		fmt.Println(QuestionQ)
		return getErr
	}

	msgSlice := []sqs.Message{}
	for i := 0; i < 100000; i++ {
		num1 := rand.Intn(9) + 1
		num2 := rand.Intn(9 + 1)
		q := Question{num1, num2}
		jsonQ, _ := json.Marshal(&q)
		msg := sqs.Message{}
		body := base64.StdEncoding.EncodeToString(jsonQ)
		msg.Body = body
		msgSlice = append(msgSlice, msg)
		if len(msgSlice) == 10 {
			//the slice is ready, time to process it
			//I guess we pass it to the Serve function
		}
	}

	for i := 0; i < 250; i++ {
		var msgList []sqs.Message
		for c := 0; c < 10; c++ {
			num1 := rand.Intn(9) + 1
			num2 := rand.Intn(9 + 1)
			q := Question{num1, num2}
			jsonQ, _ := json.Marshal(&q)
			msg := sqs.Message{}
			body := base64.StdEncoding.EncodeToString(jsonQ)
			msg.Body = body
			msgList = append(msgList, msg)
		}
		wg.Add(1)
		go addToQuestionQ(msgList, *questionq)
	}
	wg.Wait()
	return nil
}

func addToQuestionQ(msgList []sqs.Message, sqsQ sqs.Queue) {
	_, respErr := sqsQ.SendMessageBatch(msgList)
	if respErr != nil {
		log.Println("ERROR:", respErr)
	}
	wg.Done()
}
{'fraction_non_alphanumeric': 0.1474701245868294, 'fraction_numerical': 0.018306636155606407, 'mean_word_length': 3.375973303670745, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1926465', 'n_tokens_mistral': 1717, 'n_tokens_neox': 1557, 'n_words': 409}