<commit_before>package main

import (
	"log"
	"sync"

	"github.com/gocql/gocql"
	"github.com/skeswa/gophr/common"
)

func main() {
	log.Println("Establishing connection to DB")
	cluster := gocql.NewCluster("gophr.dev")
	cluster.ProtoVersion = 4
	cluster.Keyspace = "gophr"
	cluster.Consistency = gocql.One
	session, err := cluster.CreateSession()
	if err != nil {
		log.Println("Connection failed to establish successfully")
		log.Fatalln(err)
	}
	defer session.Close()
	log.Println("Connection established successfully")

	log.Println("Fetching godoc metadata")
	godocMetadataList, err := fetchGodocMetadata()
	if err != nil {
		log.Println("Failed to fetch godoc metadata")
		log.Fatalln(err)
	}

	log.Println("Fetching awesome go list")
	awesomeGoIndex, err := fetchAwesomeGoList()
	if err != nil {
		log.Println("Failed to fetch awesome go list")
		log.Fatalln(err)
	}

	log.Println("Preparing to build package models")
	packageModels, err := buildPackageModels(godocMetadataList, awesomeGoIndex)
	if err != nil {
		log.Println("Failed to build package models")
		log.Fatalln(err)
	}

	log.Println("Preparing to insert packages into database")

	var wg sync.WaitGroup
	var insertPackageErrors []error

	nbConcurrentInserts := 20
	packageChan := make(chan *common.PackageModel, nbConcurrentInserts)
	for i := 0; i < nbConcurrentInserts; i++ {
		wg.Add(1)
		go func() {
			for packageModel := range packageChan {
				err := common.InsertPackage(session, packageModel)
				if err != nil {
					insertPackageErrors = append(insertPackageErrors, err)
				}
			}
			wg.Done()
		}()
	}

	for _, packageModel := range packageModels {
		packageChan <- packageModel
	}
	close(packageChan)
	wg.Wait()

	for _, insertErr := range insertPackageErrors {
		log.Println(insertErr)
	}

	log.Println("Finished inserting packages into database")
}<commit_msg>Calling re-indexer function and exiting early for testing purposes<commit_after>package main

import (
	"log"
	"os"
	"sync"

	"github.com/gocql/gocql"
	"github.com/skeswa/gophr/common"
)

func main() {
	log.Println("Establishing connection to DB")
	cluster := gocql.NewCluster("gophr.dev")
	cluster.ProtoVersion = 4
	cluster.Keyspace = "gophr"
	cluster.Consistency = gocql.One
	session, err := cluster.CreateSession()
	if err != nil {
		log.Println("Connection failed to establish successfully")
		log.Fatalln(err)
	}
	defer session.Close()
	log.Println("Connection established successfully")

	ReIndexPackageGitHubStats(session)
	os.Exit(1)

	log.Println("Fetching godoc metadata")
	godocMetadataList, err := fetchGodocMetadata()
	if err != nil {
		log.Println("Failed to fetch godoc metadata")
		log.Fatalln(err)
	}

	log.Println("Fetching awesome go list")
	awesomeGoIndex, err := fetchAwesomeGoList()
	if err != nil {
		log.Println("Failed to fetch awesome go list")
		log.Fatalln(err)
	}

	log.Println("Preparing to build package models")
	packageModels, err := buildPackageModels(godocMetadataList, awesomeGoIndex)
	if err != nil {
		log.Println("Failed to build package models")
		log.Fatalln(err)
	}

	log.Println("Preparing to insert packages into database")

	var wg sync.WaitGroup
	var insertPackageErrors []error

	nbConcurrentInserts := 20
	packageChan := make(chan *common.PackageModel, nbConcurrentInserts)
	for i := 0; i < nbConcurrentInserts; i++ {
		wg.Add(1)
		go func() {
			for packageModel := range packageChan {
				err := common.InsertPackage(session, packageModel)
				if err != nil {
					insertPackageErrors = append(insertPackageErrors, err)
				}
			}
			wg.Done()
		}()
	}

	for _, packageModel := range packageModels {
		packageChan <- packageModel
	}
	close(packageChan)
	wg.Wait()

	for _, insertErr := range insertPackageErrors {
		log.Println(insertErr)
	}

	log.Println("Finished inserting packages into database")
	// TODO run star updater
}
{'fraction_non_alphanumeric': 0.11773940345368916, 'fraction_numerical': 0.0028780743066457353, 'mean_word_length': 4.461428571428572, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6017858', 'n_tokens_mistral': 1392, 'n_tokens_neox': 1275, 'n_words': 309}