<commit_before>package main

import (
	"bytes"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"log"
	//"os"
	"net/http"
	"strings"
	"time"
)

type GoPackage struct {
	Description    string
	GitHubURL      string
	GoDocURL       string
	IndexTime      string
	HttpStatusCode int
}

func main() {
	log.Println("Begining download")
	doc, err := goquery.NewDocument("https://godoc.org/-/index")

	if err != nil {
		log.Fatal(err)
	}

	log.Println("Finished download")
	var goPackageList = make([]*GoPackage, 0)

	// For each tr element on the page
	doc.Find("tr").Each(func(i int, s *goquery.Selection) {
		children := s.Children()
		goPackage := new(GoPackage)

		// For each child in the tr element
		children.Each(func(i int, s2 *goquery.Selection) {
			childURL, childURLexists := s.Find("a").Attr("href")
			childDescription := s.Text()

			if childURLexists == true {
				goPackage.GitHubURL = childURL
			}

			if len(childDescription) > 0 {
				// TODO check if description isn't just the url, if so dont set it
				goPackage.Description = childDescription
			}
		})

		// Only continue if this goPackage contains a GitHub URL
		// TODO check to make sure github.com is the prefix
		if strings.Contains(goPackage.GitHubURL, "github.com") {
			// Build go doc url
			goPackage.GoDocURL = ("https://godoc.org/" + goPackage.GitHubURL)

			// Create Index Time
			t := time.Now()
			time := t.String()
			goPackage.IndexTime = time

			// Add goPackage to goPackageList
			goPackageList = append(goPackageList, goPackage)
		}
	})

	for _, goPackage := range goPackageList {
		resp, err := http.Get("http://www." + goPackage.GitHubURL)
		if err != nil {
			fmt.Println("ERROR")
		}
		defer resp.Body.Close()

		// TODO Check resp status code
		goPackage.HttpStatusCode = resp.StatusCode

		_, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			panic(err)
		}
		// Call Router Functionality to Parse for array of versions

	}

	log.Println("Finished Building GoPackages")
	log.Println("Creating JSON dump")
	createJSONDump(goPackageList)
}

// Create a tmp JSON dump of all serialized goPackageData
func createJSONDump(goPackageList []*GoPackage) {
	var buffer bytes.Buffer

	for _, goPackage := range goPackageList {
		buffer.WriteString("{\"url\": \"" + goPackage.GitHubURL + "\", \"description\": \"" + goPackage.Description + "\", \"index_time\": \"" + goPackage.IndexTime + "\" },\n")
	}

	t := time.Now()
	time := t.String()
	jsonData := []byte(buffer.String())
	_ = ioutil.WriteFile("./"+time+"tmp.json", jsonData, 0644)
}

func printGoPackage(gp *GoPackage) {
	fmt.Println("GitHub URL = ", gp.GitHubURL)
	fmt.Println("Description = ", gp.Description)
	fmt.Println("Index time = ", gp.IndexTime)
	fmt.Println("GoDocURL = ", gp.GoDocURL, "\n")
}<commit_msg>Now intersect with AwesomeGo and label AwesomeGo packages<commit_after>package main

import (
	"bytes"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"log"
	//"os"
	"net/http"
	"strings"
	"time"
)

type GoPackage struct {
	Description    string
	GitHubURL      string
	GoDocURL       string
	IndexTime      string
	HttpStatusCode int
	AwesomeGo      bool
}

const (
	refsFetchURLTemplate = "https://%s.git/info/refs?service=git-upload-pack"
)

func main() {
	log.Println("Started Download of Godoc/index")

	doc, err := goquery.NewDocument("https://godoc.org/-/index")
	if err != nil {
		log.Fatal(err)
	}

	log.Println("Finished Download of Godoc/index")
	log.Println("Started scraping all github packages from Godoc/index")

	var goPackageList = make([]*GoPackage, 0)
	var goPackageMap = make(map[string]*GoPackage)

	// For each tr element on the page
	doc.Find("tr").Each(func(i int, s *goquery.Selection) {
		children := s.Children()
		goPackage := new(GoPackage)

		// For each child in the tr element
		children.Each(func(i int, s2 *goquery.Selection) {
			childURL, childURLexists := s.Find("a").Attr("href")
			childDescription := s.Text()

			if childURLexists == true {
				goPackage.GitHubURL = childURL
			}

			if len(childDescription) > 0 {
				// TODO check if description isn't just the url, if so dont set it
				goPackage.Description = childDescription
			}
		})

		// Only continue if this goPackage contains a GitHub URL
		// TODO check to make sure github.com is the prefix
		if strings.Contains(goPackage.GitHubURL, "github.com") {
			// Build go doc url
			goPackage.GoDocURL = ("https://godoc.org/" + goPackage.GitHubURL)

			// Create Index Time
			t := time.Now()
			time := t.String()
			goPackage.IndexTime = time

			// Add goPackage to goPackageList
			goPackageList = append(goPackageList, goPackage)
			// Hash the goPackage.GitHubURL for lookup
			goPackageMap[goPackage.GitHubURL] = goPackage
		}
	})

	log.Println("Finished scraping all github packages from Godoc/index")
	log.Println("Started Download of awesome-go/README.md")

	doc, err = goquery.NewDocument("https://godoc.org/-/index")
	if err != nil {
		log.Fatal(err)
	}

	log.Println("Finished Download of awesome-go/README.md")
	log.Println("Started scraping awesome-go/README.md")

	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		childURL, _ := s.Attr("href")

		goPackage, goPackageExists := goPackageMap[childURL]

		if goPackageExists {
			goPackage.AwesomeGo = true
			goPackageMap[childURL] = goPackage
			printGoPackage(goPackage)
		}
	})

	log.Println("Finished scraping awesome-go/README.md")

	for _, goPackage := range goPackageMap {
		resp, err := http.Get("http://www.example.com")
		if err != nil {
			fmt.Println("ERROR")
		}
		defer resp.Body.Close()

		// TODO Check resp status code
		goPackage.HttpStatusCode = resp.StatusCode

		_, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			panic(err)
		}
		// Call Router Functionality to Parse for array of versions

	}

	log.Println("Finished Building GoPackages")
	log.Println("Creating JSON dump")
	createJSONDump(goPackageList)
}

// Create a tmp JSON dump of all serialized goPackageData
func createJSONDump(goPackageList []*GoPackage) {
	var buffer bytes.Buffer

	for _, goPackage := range goPackageList {
		buffer.WriteString("{\"url\": \"" + goPackage.GitHubURL + "\", \"description\": \"" + goPackage.Description + "\", \"index_time\": \"" + goPackage.IndexTime + "\" },\n")
	}

	t := time.Now()
	time := t.String()
	jsonData := []byte(buffer.String())
	_ = ioutil.WriteFile("./"+time+"tmp.json", jsonData, 0644)
}

func printGoPackage(gp *GoPackage) {
	fmt.Println("GitHub URL = ", gp.GitHubURL)
	fmt.Println("Description = ", gp.Description)
	fmt.Println("Index time = ", gp.IndexTime)
	fmt.Println("GoDocURL = ", gp.GoDocURL)
	fmt.Println("AwesomeGo = ", gp.AwesomeGo, "\n")
}
{'fraction_non_alphanumeric': 0.1428141006327207, 'fraction_numerical': 0.0021090689966857487, 'mean_word_length': 3.626480836236934, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4892249', 'n_tokens_mistral': 2483, 'n_tokens_neox': 2237, 'n_words': 542}