<commit_before>package common

import (
	"log"
	"time"
)

type GitHubAPIKeyChain struct {
	GitHubAPIKeys []GitHubAPIKeyModel
	CurrentKey    GitHubAPIKeyModel
}

func NewGitHubAPIKeyChain() *GitHubAPIKeyChain {
	log.Println("CREATING NEW KEYCHAIN")
	newGitHubAPIKeyChain := GitHubAPIKeyChain{}
	newGitHubAPIKeyChain.GitHubAPIKeys = make([]GitHubAPIKeyModel, 0)

	keys := []string{}

	// For each api key create a new key model and push to the the list of GitHubAPIKeys
	for _, key := range keys {
		log.Println("KEY =", key)
		gitHubAPIKeyModel := GitHubAPIKeyModel{
			string(key),
			5000,
			5000,
			time.Time{},
		}
		log.Println("Priming API KEY")
		gitHubAPIKeyModel.prime()
		gitHubAPIKeyModel.print()

		newGitHubAPIKeyChain.GitHubAPIKeys = append(newGitHubAPIKeyChain.GitHubAPIKeys, gitHubAPIKeyModel)
	}

	//TODO sort
	newGitHubAPIKeyChain.setCurrentKey()
	return &newGitHubAPIKeyChain
}

func (gitHubAPIKeyChain *GitHubAPIKeyChain) getAPIKeyModel() *GitHubAPIKeyModel {
	if gitHubAPIKeyChain.CurrentKey.RemainingUses > 0 {
		return &gitHubAPIKeyChain.CurrentKey
	}
	log.Println("KEY NEEDS TO BE SWAPED")
	newGitHubAPIKeys := make([]GitHubAPIKeyModel, 0)
	var swapModel GitHubAPIKeyModel
	for index, APIKeyModel := range gitHubAPIKeyChain.GitHubAPIKeys {
		if index == 0 {
			swapModel = APIKeyModel
		} else {
			newGitHubAPIKeys = append(newGitHubAPIKeys, APIKeyModel)
		}
	}
	newGitHubAPIKeys = append(newGitHubAPIKeys, swapModel)
	gitHubAPIKeyChain.GitHubAPIKeys = newGitHubAPIKeys
	gitHubAPIKeyChain.setCurrentKey()

	if gitHubAPIKeyChain.CurrentKey.RemainingUses == 0 {
		coolDown(gitHubAPIKeyChain.CurrentKey)
	}

	return &gitHubAPIKeyChain.CurrentKey
}

func (gitHubAPIKeyChain *GitHubAPIKeyChain) setCurrentKey() {
	if len(gitHubAPIKeyChain.GitHubAPIKeys) == 0 {
		gitHubAPIKeyChain.CurrentKey = GitHubAPIKeyModel{}
	}

	gitHubAPIKeyChain.CurrentKey = gitHubAPIKeyChain.GitHubAPIKeys[0]

}

func coolDown(apiKeyModel GitHubAPIKeyModel) {
	timeNow := time.Now()
	log.Println("Time now = ", timeNow)
	resetTime := apiKeyModel.RateLimitResetTime
	log.Println("Reset time = ", resetTime)

	sleepTime := resetTime.Sub(timeNow)
	log.Println("Sleep time is = ", sleepTime)
	time.Sleep(sleepTime)
}<commit_msg>clean up and fix weird bug<commit_after>package common

import (
	"log"
	"time"
)

type GitHubAPIKeyChain struct {
	GitHubAPIKeys []GitHubAPIKeyModel
	CurrentKey    GitHubAPIKeyModel
}

func NewGitHubAPIKeyChain() *GitHubAPIKeyChain {
	log.Println("CREATING NEW KEYCHAIN")
	newGitHubAPIKeyChain := GitHubAPIKeyChain{}

	// TODO QUERY FOR KEYS
	keys := []string{}

	newGitHubAPIKeyChain.GitHubAPIKeys = initializeGitHubAPIKeys(keys)

	//TODO sort
	newGitHubAPIKeyChain.setCurrentKey()
	return &newGitHubAPIKeyChain
}

// For each api key create a new key model and push to the the list of GitHubAPIKeys
func initializeGitHubAPIKeys(keys []string) []GitHubAPIKeyModel {
	var gitHubAPIKeyModels = make([]GitHubAPIKeyModel, 0)

	for _, key := range keys {
		log.Println("KEY =", key)
		gitHubAPIKeyModel := GitHubAPIKeyModel{
			string(key),
			5000,
			5000,
			time.Time{},
		}
		log.Println("Priming API KEY")
		gitHubAPIKeyModel.prime()
		gitHubAPIKeyModel.print()

		gitHubAPIKeyModels = append(gitHubAPIKeyModels, gitHubAPIKeyModel)
	}

	return gitHubAPIKeyModels
}

func (gitHubAPIKeyChain *GitHubAPIKeyChain) getAPIKeyModel() *GitHubAPIKeyModel {
	if gitHubAPIKeyChain.CurrentKey.RemainingUses > 0 {
		return &gitHubAPIKeyChain.CurrentKey
	}

	log.Println("KEY NEEDS TO BE SWAPED")
	gitHubAPIKeyChain.shuffleKeys()
	gitHubAPIKeyChain.setCurrentKey()
	gitHubAPIKeyChain.CurrentKey.prime()

	// If new key's remaining use is 0 set a time out
	if gitHubAPIKeyChain.CurrentKey.RemainingUses <= 0 {
		setRequestTimout(gitHubAPIKeyChain.CurrentKey)
	}

	return &gitHubAPIKeyChain.CurrentKey
}

func (gitHubAPIKeyChain *GitHubAPIKeyChain) shuffleKeys() {
	newGitHubAPIKeys := make([]GitHubAPIKeyModel, 0)
	var firstAPIModelInArray GitHubAPIKeyModel

	for index, APIKeyModel := range gitHubAPIKeyChain.GitHubAPIKeys {
		if index == 0 {
			firstAPIModelInArray = APIKeyModel
		} else {
			newGitHubAPIKeys = append(newGitHubAPIKeys, APIKeyModel)
		}
	}
	newGitHubAPIKeys = append(newGitHubAPIKeys, firstAPIModelInArray)

	gitHubAPIKeyChain.GitHubAPIKeys = newGitHubAPIKeys
}

func (gitHubAPIKeyChain *GitHubAPIKeyChain) setCurrentKey() {
	if len(gitHubAPIKeyChain.GitHubAPIKeys) == 0 {
		gitHubAPIKeyChain.CurrentKey = GitHubAPIKeyModel{}
	}

	gitHubAPIKeyChain.CurrentKey = gitHubAPIKeyChain.GitHubAPIKeys[0]
}

func setRequestTimout(apiKeyModel GitHubAPIKeyModel) {
	timeNow := time.Now()
	log.Printf("The current time is %s. \n", timeNow)
	resetTime := apiKeyModel.RateLimitResetTime
	log.Printf("APIKey Reset time is %s. \n", resetTime)
	sleepTime := resetTime.Sub(timeNow)
	log.Printf("Indexer will sleep for %s. \n", sleepTime)
	time.Sleep(sleepTime)
}
{'fraction_non_alphanumeric': 0.09643003693065244, 'fraction_numerical': 0.006360279031596225, 'mean_word_length': 5.578947368421052, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3196767', 'n_tokens_mistral': 1867, 'n_tokens_neox': 1693, 'n_words': 336}