<commit_before>package main

import (
	"crypto/rand"
	"fmt"
	. "github.com/sendgrid/solr-go"
	"io"
	"time"
)

var (
	solrHttpRetrier SolrHTTP
)

func init() {
	var err error
	solrZk := NewSolrZK("zk:2181", "solr", "solrtest")
	err = solrZk.Listen()
	if err != nil {
		panic(err)
	}
	solrHttp, err := NewSolrHTTP(solrZk, "solrtest", User("solr"), Password("<PASSWORD>"), MinRF(2))
	if err != nil {
		panic(err)
	}
	solrHttpRetrier = NewSolrHttpRetrier(solrHttp, 5, 100*time.Millisecond)

}
func main() {
	const limit int = 1000 * 10
	numFound, err := run(limit)
	if err != nil {
		panic(err)
	}
	if limit != int(numFound) {
		panic(fmt.Sprintf("limit did not match what was found %d=%d", limit, numFound))
	}
	fmt.Println(fmt.Sprintf("runner done %d", numFound))
}

func run(limit int) (uint32, error) {
	uuid, _ := newUUID()
	for i := 0; i < limit; i++ {
		shardKey := "mycrazyshardkey" + string(i%10)
		iterationId, _ := newUUID()
		doc := map[string]interface{}{
			"id":         shardKey + "!rando" + iterationId,
			"email":      "rando" + iterationId + "@<EMAIL>",
			"first_name": "tester" + iterationId,
			"last_name":  uuid,
		}
		if i < limit-20 {
			err := solrHttpRetrier.Update(doc["id"].(string), true, doc, Commit(false))
			if err != nil {
				fmt.Print(err)
			}
		} else {
			err := solrHttpRetrier.Update(doc["id"].(string), true, doc, Commit(true))
			if err != nil {
				fmt.Print(err)
			}
		}

	}
	r, err := solrHttpRetrier.Read(Query("*:*"), FilterQuery("last_name:"+uuid), Rows(uint32(limit)))
	return r.Response.NumFound, err

}
func newUUID() (string, error) {
	uuid := make([]byte, 16)
	n, err := io.ReadFull(rand.Reader, uuid)
	if n != len(uuid) || err != nil {
		return "", err
	}
	// variant bits; see section 4.1.1
	uuid[8] = uuid[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	uuid[6] = uuid[6]&^0xf0 | 0x40
	return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
}<commit_msg>add panics to runner<commit_after>package main

import (
	"crypto/rand"
	"fmt"
	. "github.com/sendgrid/solr-go"
	"io"
	"time"
)

var (
	solrHttpRetrier SolrHTTP
)

func init() {
	var err error
	solrZk := NewSolrZK("zk:2181", "solr", "solrtest")
	err = solrZk.Listen()
	if err != nil {
		panic(err)
	}
	solrHttp, err := NewSolrHTTP(solrZk, "solrtest", User("solr"), Password("<PASSWORD>"), MinRF(2))
	if err != nil {
		panic(err)
	}
	solrHttpRetrier = NewSolrHttpRetrier(solrHttp, 5, 100*time.Millisecond)

}
func main() {
	const limit int = 1000 * 10
	numFound, err := run(limit)
	if err != nil {
		panic(err)
	}
	if limit != int(numFound) {
		panic(fmt.Sprintf("limit did not match what was found %d=%d", limit, numFound))
	}
	fmt.Println(fmt.Sprintf("runner done %d", numFound))
}

func run(limit int) (uint32, error) {
	uuid, _ := newUUID()
	for i := 0; i < limit; i++ {
		shardKey := "mycrazyshardkey" + string(i%10)
		iterationId, _ := newUUID()
		doc := map[string]interface{}{
			"id":         shardKey + "!rando" + iterationId,
			"email":      "rando" + iterationId + "@sendgrid.com",
			"first_name": "tester" + iterationId,
			"last_name":  uuid,
		}
		if i < limit-20 {
			err := solrHttpRetrier.Update(doc["id"].(string), true, doc, Commit(false))
			if err != nil {
				panic(err)
			}
		} else {
			err := solrHttpRetrier.Update(doc["id"].(string), true, doc, Commit(true))
			if err != nil {
				panic(err)
			}
		}

	}
	r, err := solrHttpRetrier.Read(Query("*:*"), FilterQuery("last_name:"+uuid), Rows(uint32(limit)))
	return r.Response.NumFound, err

}
func newUUID() (string, error) {
	uuid := make([]byte, 16)
	n, err := io.ReadFull(rand.Reader, uuid)
	if n != len(uuid) || err != nil {
		return "", err
	}
	// variant bits; see section 4.1.1
	uuid[8] = uuid[8]&^0xc0 | 0x80
	// version 4 (pseudo-random); see section 4.1.3
	uuid[6] = uuid[6]&^0xf0 | 0x40
	return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
}
{'fraction_non_alphanumeric': 0.1952887537993921, 'fraction_numerical': 0.029381965552178316, 'mean_word_length': 3.7982989064398542, 'pattern_counts': {'":': 8, '<': 10, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '881471', 'n_tokens_mistral': 1762, 'n_tokens_neox': 1640, 'n_words': 359}