<commit_before>package main

import (
	"fmt"
	"github.com/swpecht/GoMM"
)

func main() {
	numIterations := 500
	headNode := "10.0.2.15:7946" // "192.168.3.11:7946"

	f := GoMM.ClientFactory{}
	client, err := GoMM.GetTCPClient(f)
	if err != nil {
		fmt.Println("Failed to create client", err.Error())
		return
	}

	client.Start()
	client.Join(headNode)

	client.WaitActive()

	for i := 0; i < numIterations; i++ {
		id, numNodes := UpdatePool(client)
		Shuffle(id, numNodes, client)
		DoIteration(id)
	}
}

// Returns the id, numActiveNodes
func UpdatePool(client *GoMM.Client) (int, int) {
	return 0, 1
}

// Shuffles / distributes the data between all nods
func Shuffle(id, numNodes int, client *GoMM.Client) {

}

// Perform the actual iterations
func DoIteration(id int) {

}<commit_msg>Finished implementing workload bench, tested on 2 nodes.<commit_after>package main

import (
	"fmt"
	"github.com/swpecht/GoMM"
	"time"
)

func main() {
	numIterations := 50
	headNode := "10.0.2.15:7946" // "192.168.3.11:7946"

	f := GoMM.ClientFactory{}
	client, err := GoMM.GetTCPClient(f)
	if err != nil {
		fmt.Println("Failed to create client", err.Error())
		return
	}

	client.Start()
	client.Join(headNode)

	client.WaitActive()
	// Check if first worker
	iter := 0
	if client.NumActiveMembers() > 1 {
		// If not, should be getting the iteration
		msg := <-client.BroadcastChannel
		iter = int(msg.FloatData[0])
	}

	id := client.GetId()
	numNodes := client.NumActiveMembers()
	for i := iter; i < numIterations; i++ {
		oldId := id
		oldNum := numNodes

		UpdatePool(client, i)

		id = client.GetId()
		numNodes = client.NumActiveMembers()

		// Shuffle if update or first iteration
		if oldId != id || oldNum != numNodes || i == iter {
			// Need to reshuffle data
			Shuffle(id, numNodes, client)
		}
		DoIteration(id, numNodes, i, client)
	}
}

// Returns the current iteration
func UpdatePool(client *GoMM.Client, iter int) int {
	if client.NumMembers() != client.NumActiveMembers() {
		client.UpdateActiveMembers()
		// If client 0, sent the iteration broadcast to everyone
		if client.GetId() == 0 {
			client.Broadcast([]string{}, []float64{float64(iter)})
		}
		// Receive the iteration broadcast
		msg := <-client.BroadcastChannel
		iter = int(msg.FloatData[0])

	}
	fmt.Println("Doing UpdatePool barrier")
	client.Barrier()
	return iter
}

// Shuffles / distributes the data between all nods
func Shuffle(id, numNodes int, client *GoMM.Client) {
	// Do a barrier for worst case coordination
	fmt.Println("Doing Shuffle barrier")
	client.Barrier()
}

// Perform the actual iterations
func DoIteration(id, numNodes, iter int, client *GoMM.Client) {
	duration := time.Duration(500 / numNodes)
	time.Sleep(duration * time.Millisecond)
	// Assume worst case communications
	// client.Barrier()
	fmt.Printf("Iteration %v \n", iter)
}
{'fraction_non_alphanumeric': 0.13968253968253969, 'fraction_numerical': 0.02398589065255732, 'mean_word_length': 3.8067796610169493, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4288140', 'n_tokens_mistral': 1079, 'n_tokens_neox': 984, 'n_words': 266}