<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/jpfairbanks/cse6140/project/hashes"
	"log"
	"math/rand"
	"time"
)

//CMSketch: A streaming data structure for tracking the elements of
//a long vector in a space efficient manner.
//Provides update and query operations
type CMSketch struct {
	Depth   int64
	Width   int64
	Hash    []hashes.Hash
	Counter Matrix
}

//NewCMSketch: Allocates a new sketch
//Does not create the hashes. We leave that up to you so that
//the consumer can pick hashes according to different schema.
func NewCMSketch(Depth, Width int64) CMSketch {
	Hash := make([]hashes.Hash, Depth)
	Counter := NewMatrix(Depth, Width)
	cms := CMSketch{Depth, Width, Hash, Counter}
	return cms
}

//Clone: Return a copy of the skecth with the same parameters and hashes,
//but a fresh set of counters. Suggested use is to compare two different
//manipulations for accuracy.
func (cms *CMSketch) Clone() CMSketch {
	Counter := NewMatrix(cms.Depth, cms.Width)
	out := CMSketch{cms.Depth, cms.Width, cms.Hash, Counter}
	return out
}

//Equal: Tell if two cms instances have the same parameters and data.
func (cms *CMSketch) Equal(other *CMSketch) bool {
	depths := cms.Depth == other.Depth
	widths := cms.Width == other.Width
	var hash bool
	hash = true
	for i, h := range cms.Hash {
		hash = other.Hash[i].Equal(h) && hash
		if !hash {
			break
		}
	}
	counters := cms.Counter.Equal(&other.Counter)
	out := depths && widths && hash && counters
	return out
}

//RandomHashes: Make uniformly random hashes from a user provided
//stream of random numbers.
func RandomHashes(r *rand.Rand, Depth int64) []hashes.Hash {
	hslice := make([]hashes.Hash, Depth)
	for i, _ := range hslice {
		hslice[i] = hashes.Rand(r)
	}
	return hslice
}

//UpdateSerial: Insert a single item into the sketch with a count.
//If counts can be negative, then you must estimate differently
func (cms *CMSketch) UpdateSerial(position int64, count int64) {
	for i, h := range cms.Hash {
		cms.Counter.Add(int64(i), h.Apply(position)%cms.Width, count)
	}
}

//AddSignalSlow: Updates the counter and signals completion on a channel
func (cms *CMSketch) AddSignalSlow(row int64, position int64,
	count int64, ch chan int64) {
	h := cms.Hash[row]
	cms.Counter.Add(row, h.Apply(position)%cms.Width, count)
	ch <- 1
}

//UpdateDepthParallelSlow: Insert a single item into the sketch with a count.
func (cms *CMSketch) UpdateDepthParallelSlow(position int64, count int64) {
	ch := make(chan int64, cms.Depth)
	var i int64
	for i = 0; i < cms.Depth; i++ {
		go cms.AddSignalSlow(i, position, count, ch)
	}
	for i = 0; i < cms.Depth; i++ {
		<-ch
	}
}

//AddSignal: Updates the counter and signals completion on a channel
func (cms *CMSketch) AddSignal(start_row int64, end_row int64, position int64,
	count int64, ch chan int64) {
	var h hashes.Hash
	for row := start_row; row < end_row; row++ {
		h = cms.Hash[row]
		cms.Counter.Add(row, h.Apply(position)%cms.Width, count)
	}
	ch <- 1
}

//UpdateDepthParallel: Insert a single item into the sketch with a count.
func (cms *CMSketch) UpdateDepthParallel(position int64, count int64, numProcs int64) {
	var i int64
	NbyP := cms.Depth / numProcs
	ch := make(chan int64, numProcs)
	for i = 0; i < numProcs; i++ {
		go cms.AddSignal(i*NbyP, (i+1)*NbyP, position, count, ch)
	}
	for i = 0; i < numProcs; i++ {
		<-ch
	}
}

//BatchUpdate: insert a batch of edges all at once.
//You must wait for a signal on the channel in order to ensure correct results
func (cms *CMSketch) BatchUpdate(elements []int64, ch chan int64, numProcs int64) {
	for _, z := range elements {
		cms.UpdateDepthParallel(z, 1, numProcs)
	}
	ch <- 1
}

//BatchUpdateSort: insert a batch of edges all at once.
//You must wait for a signal on the channel in order to ensure correct results
//This method uses a sort to improve cache locality.
func (cms *CMSketch) BatchUpdateSort(elements []int64, ch chan int64, numProcs int64) {
	//TODO: this method does not actually do any sorting
	for _, z := range elements {
		cms.UpdateDepthParallel(z, 1, numProcs)
	}
	ch <- 1
}

//PointQuery: query the value at position
func (cms *CMSketch) PointQuery(position int64) int64 {<commit_msg>fixing the batch methods so they actually do something. Without tests<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/jpfairbanks/cse6140/project/hashes"
	"log"
	"math/rand"
	"time"
)

//CMSketch: A streaming data structure for tracking the elements of
//a long vector in a space efficient manner.
//Provides update and query operations
type CMSketch struct {
	Depth   int64
	Width   int64
	Hash    []hashes.Hash
	Counter Matrix
}

//NewCMSketch: Allocates a new sketch
//Does not create the hashes. We leave that up to you so that
//the consumer can pick hashes according to different schema.
func NewCMSketch(Depth, Width int64) CMSketch {
	Hash := make([]hashes.Hash, Depth)
	Counter := NewMatrix(Depth, Width)
	cms := CMSketch{Depth, Width, Hash, Counter}
	return cms
}

//Clone: Return a copy of the skecth with the same parameters and hashes,
//but a fresh set of counters. Suggested use is to compare two different
//manipulations for accuracy.
func (cms *CMSketch) Clone() CMSketch {
	Counter := NewMatrix(cms.Depth, cms.Width)
	out := CMSketch{cms.Depth, cms.Width, cms.Hash, Counter}
	return out
}

//Equal: Tell if two cms instances have the same parameters and data.
func (cms *CMSketch) Equal(other *CMSketch) bool {
	depths := cms.Depth == other.Depth
	widths := cms.Width == other.Width
	var hash bool
	hash = true
	for i, h := range cms.Hash {
		hash = other.Hash[i].Equal(h) && hash
		if !hash {
			break
		}
	}
	counters := cms.Counter.Equal(&other.Counter)
	out := depths && widths && hash && counters
	return out
}

//RandomHashes: Make uniformly random hashes from a user provided
//stream of random numbers.
func RandomHashes(r *rand.Rand, Depth int64) []hashes.Hash {
	hslice := make([]hashes.Hash, Depth)
	for i, _ := range hslice {
		hslice[i] = hashes.Rand(r)
	}
	return hslice
}

//UpdateSerial: Insert a single item into the sketch with a count.
//If counts can be negative, then you must estimate differently
func (cms *CMSketch) UpdateSerial(position int64, count int64) {
	for i, h := range cms.Hash {
		cms.Counter.Add(int64(i), h.Apply(position)%cms.Width, count)
	}
}

//AddSignalSlow: Updates the counter and signals completion on a channel
func (cms *CMSketch) AddSignalSlow(row int64, position int64,
	count int64, ch chan int64) {
	h := cms.Hash[row]
	cms.Counter.Add(row, h.Apply(position)%cms.Width, count)
	ch <- 1
}

//UpdateDepthParallelSlow: Insert a single item into the sketch with a count.
func (cms *CMSketch) UpdateDepthParallelSlow(position int64, count int64) {
	ch := make(chan int64, cms.Depth)
	var i int64
	for i = 0; i < cms.Depth; i++ {
		go cms.AddSignalSlow(i, position, count, ch)
	}
	for i = 0; i < cms.Depth; i++ {
		<-ch
	}
}

//AddSignal: Updates the counter and signals completion on a channel
func (cms *CMSketch) AddSignal(start_row int64, end_row int64, position int64,
	count int64, ch chan int64) {
	var h hashes.Hash
	for row := start_row; row < end_row; row++ {
		h = cms.Hash[row]
		cms.Counter.Add(row, h.Apply(position)%cms.Width, count)
	}
	ch <- 1
}

//UpdateDepthParallel: Insert a single item into the sketch with a count.
func (cms *CMSketch) UpdateDepthParallel(position int64, count int64, numProcs int64) {
	var i int64
	NbyP := cms.Depth / numProcs
	ch := make(chan int64, numProcs)
	for i = 0; i < numProcs; i++ {
		go cms.AddSignal(i*NbyP, (i+1)*NbyP, position, count, ch)
	}
	for i = 0; i < numProcs; i++ {
		<-ch
	}
}

//BatchUpdate: insert a batch of edges all at once.
//You must wait for a signal on the channel in order to ensure correct results
func (cms *CMSketch) BatchUpdate(elements []int64, ch chan int64, numProcs int64) {
	batchSize := len(elements)
	for i, h := range cms.Hash {
		yarr := make([]int64, batchSize)
		h.ApplyBatch(elements, yarr)
		for k, y := range yarr {
			yarr[k] = y % cms.Width
		}
		for _, y := range yarr {
			cms.Counter.Add(int64(i), y, 1)
		}
	}
	ch <- 1
}

//BatchUpdateSort: insert a batch of edges all at once.
//You must wait for a signal on the channel in order to ensure correct results
//This method uses a sort to improve cache locality.
func (cms *CMSketch) BatchUpdateSort(elements []int64, ch chan int64, numProcs int64) {
	//TODO: this method needs to be optimized separately from BatchUpdate
	batchSize := len(elements)
	for i, h := range cms.Hash {
		yarr := make([]int64, batchSize)
		h.ApplyBatchSort(elements, yarr)
		for k, y := range yarr {
			yarr[k] = y % cms.Width
		}
		for _, y := range yarr {
			cms.Counter.Add(int64(i), y, 1)
		}
	}
	ch <- 1
}

//PointQuery: query the value at position
func (cms *CMSketch) PointQuery(position int64) int64 {
{'fraction_non_alphanumeric': 0.11540648095508811, 'fraction_numerical': 0.019329164297896533, 'mean_word_length': 3.840946615299945, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4353235', 'n_tokens_mistral': 3309, 'n_tokens_neox': 3038, 'n_words': 1099}