<commit_before>package main

import (
	"log"
	"sync"
	"time"

	"github.com/gocql/gocql"
	"github.com/skeswa/gophr/common"
)

func ReIndexPackageGitHubStats(session *gocql.Session) {
	log.Println("Starting ReIndexing PackageModel GitHub Data")
	packageModels, err := common.ScanAllPackageModels(session)
	log.Println(err)
	log.Printf("%d packages found", len(packageModels))

	totalNumPackages := len(packageModels)
	log.Printf("Total num packages found = %d ", totalNumPackages)

	log.Println("Initializing GitHub Component")
	gitHubRequestService := common.NewGitHubRequestService()

	var wg sync.WaitGroup
	nbConcurrentInserts := 20
	packageChan := make(chan common.GitHubPackageModelDTO, 20)

	log.Printf("Spinning up %d consumers", nbConcurrentInserts)
	for i := 0; i < nbConcurrentInserts; i++ {
		wg.Add(1)
		go func() {
			for gitHubPackageModelDTO := range packageChan {
				packageStarCount := parseStarCount(gitHubPackageModelDTO.ResponseBody)
				indexTime := time.Now()
				log.Printf("StarCount %d \n", packageStarCount)
				log.Printf("New index time %s \n", indexTime)
				packageModel := gitHubPackageModelDTO.Package
				packageModel.IndexTime = &indexTime
				packageModel.Stars = &packageStarCount
				err := common.InsertPackage(session, &packageModel)
				if err != nil {
					log.Println("ERRORRRRRR \n\n\n\n")
					log.Println(err)
				}
				log.Println("Index time has been updated")
				log.Printf("%+v\n", packageModel)
				// TODO save data to DB

			}
			wg.Done()
		}()
	}

	log.Printf("Preparing to fetch stars for %d repos", totalNumPackages)
	for count, packageModel := range packageModels {
		log.Printf("PROCESSING PACKAGE %s/%s #%d \n", *packageModel.Author, *packageModel.Repo, count)
		packageModelGitHubData, err := gitHubRequestService.FetchGitHubDataForPackageModel(*packageModel)
		if err != nil {
			log.Printf("PANIC %v \n\n\n\n\n\n\n", err)
		}
		packageChan <- common.GitHubPackageModelDTO{Package: *packageModel, ResponseBody: packageModelGitHubData}
		time.Sleep(100 * time.Millisecond)
	}

	close(packageChan)
	wg.Wait()
	log.Println("Finished testing star fetching")
}

func parseStarCount(responseBody map[string]interface{}) int {
	starCount := responseBody["stargazers_count"]
	if starCount == nil {
		return 0
	}

	return int(starCount.(float64))
}<commit_msg>reduced request time buffer<commit_after>package main

import (
	"log"
	"sync"
	"time"

	"github.com/gocql/gocql"
	"github.com/skeswa/gophr/common"
)

func ReIndexPackageGitHubStats(session *gocql.Session) {
	log.Println("Starting ReIndexing PackageModel GitHub Data")
	packageModels, err := common.ScanAllPackageModels(session)
	log.Println(err)
	log.Printf("%d packages found", len(packageModels))

	totalNumPackages := len(packageModels)
	log.Printf("Total num packages found = %d ", totalNumPackages)

	log.Println("Initializing GitHub Component")
	gitHubRequestService := common.NewGitHubRequestService()

	var wg sync.WaitGroup
	nbConcurrentInserts := 20
	packageChan := make(chan common.GitHubPackageModelDTO, 20)

	log.Printf("Spinning up %d consumers", nbConcurrentInserts)
	for i := 0; i < nbConcurrentInserts; i++ {
		wg.Add(1)
		go func() {
			for gitHubPackageModelDTO := range packageChan {
				packageStarCount := common.ParseStarCount(gitHubPackageModelDTO.ResponseBody)
				indexTime := time.Now()
				log.Printf("StarCount %d \n", packageStarCount)
				log.Printf("New index time %s \n", indexTime)
				packageModel := gitHubPackageModelDTO.Package
				packageModel.IndexTime = &indexTime
				packageModel.Stars = &packageStarCount
				err := common.InsertPackage(session, &packageModel)
				if err != nil {
					log.Println(err)
				}
				log.Println("Index time has been updated")
				log.Printf("%+v\n", packageModel)
			}
			wg.Done()
		}()
	}

	log.Printf("Preparing to fetch stars for %d repos", totalNumPackages)
	for count, packageModel := range packageModels {
		log.Printf("PROCESSING PACKAGE %s/%s #%d \n", *packageModel.Author, *packageModel.Repo, count)
		packageModelGitHubData, err := gitHubRequestService.FetchGitHubDataForPackageModel(*packageModel)
		if err != nil {
			log.Printf("PANIC %v \n\n\n\n\n\n\n", err)
		}
		packageChan <- common.GitHubPackageModelDTO{Package: *packageModel, ResponseBody: packageModelGitHubData}
		time.Sleep(50 * time.Millisecond)
	}

	close(packageChan)
	wg.Wait()
	log.Println("Finished testing star fetching")
}
{'fraction_non_alphanumeric': 0.12451627589346688, 'fraction_numerical': 0.004552697473252903, 'mean_word_length': 4.766404199475065, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3187715', 'n_tokens_mistral': 1648, 'n_tokens_neox': 1478, 'n_words': 309}