<commit_before>package main

import (
	"fmt"
	"jbossinfo"
	"net/http"
	"time"
)

var (
	JbossStatusUrls = map[string]string{
		"TST": "http://127.0.0.1:8080/status?XML=true",
	}
)

func getCurrentInfo(site string) *jbossinfo.JbossStatus {
	fmt.Printf("Trying to get xml from: %s\n", JbossStatusUrls[site])

	resp, respErr := http.Get(JbossStatusUrls[site])
	if respErr != nil {
		fmt.Printf("Can't get jboss xml: %s\n", respErr)
		return nil
	}

	info, infoErr := jbossinfo.ParseJbossInfoXML(resp.Body)
	if infoErr != nil {
		fmt.Printf("Parsing jboss xml failed: %s\n", infoErr)
		return nil
	}

	return info
}

func pullJboss(sites []string, interval time.Duration) {

	queue := make(chan *InsertRequest, 100)
	ok := sqliteWriteHandler(queue)

	ticker := time.Tick(interval)

	go func() {
		for now := range ticker {
			for _, site := range sites {
				info := getCurrentInfo(site)

				queue <- &InsertRequest{site, now, info}
			}
		}
		queue <- nil
	}()
	<-ok
}<commit_msg>jboss2sqlite, fix pulling into sqlite<commit_after>package main

import (
	"fmt"
	"jbossinfo"
	"net/http"
	"time"
)

var (
	JbossStatusUrls = map[string]string{
		"TST": "http://127.0.0.1:8080/status?XML=true",
	}
)

func getCurrentInfo(site string) *jbossinfo.JbossStatus {
	fmt.Printf("Trying to get xml from: %s\n", JbossStatusUrls[site])

	resp, respErr := http.Get(JbossStatusUrls[site])
	if respErr != nil {
		fmt.Printf("Can't get jboss xml: %s\n", respErr)
		return nil
	}

	info, infoErr := jbossinfo.ParseJbossInfoXML(resp.Body)
	if infoErr != nil {
		fmt.Printf("Parsing jboss xml failed: %s\n", infoErr)
		return nil
	}

	return info
}

func pullSite(site string, now time.Time, queue chan *InsertRequest) {
	fmt.Printf("pulling %s: %s\n", site, now.Format(time.RFC3339))
	info := getCurrentInfo(site)

	queue <- &InsertRequest{site, now, info}
}

func pullJboss(sites []string, interval time.Duration) {
	for _, site := range sites {
		if JbossStatusUrls[site] == "" {
			fmt.Printf("Error, %s is an unkonwn site\n", site)
			return
		}
	}

	ticker := time.Tick(interval)

	for _ = range ticker {
		queue := make(chan *InsertRequest, 10)
		ok := sqliteWriteHandler(queue)

    // remember to parallelize pullSite, but remember that the sqlite close has to come after all are done
		for _, site := range sites {
			pullSite(site, time.Now(), queue)
		}

		queue <- nil
		<-ok
	}
}
{'fraction_non_alphanumeric': 0.14641350210970464, 'fraction_numerical': 0.012658227848101266, 'mean_word_length': 3.9190871369294604, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2620408', 'n_tokens_mistral': 991, 'n_tokens_neox': 908, 'n_words': 218}