<commit_before>package main

import (
	"fmt"
	"golang.org/x/net/html"
	"net/http"
	"net/url"
	"os"
	"strings"
)

// Helper function to pull the href attribute from a Token
func getHref(t html.Token) (ok bool, href string) {
	// Iterate over all of the Token's attributes until we find an "href"
	for _, a := range t.Attr {
		if a.Key == "href" {
			href = a.Val
			ok = true
		}
	}

	// "bare" return will return the variables (ok, href) as defined in
	// the function definition
	return
}

type UrlResponse struct {
	from string
	url  string
	code int
	err  error
}

type NewUrl struct {
	from string
	url  string
}

func crawl(chWork chan NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	for true {
		new := <-chWork
		crawlOne(new, ch, chFinished)
	}
}

// Extract all http** links from a given webpage
func crawlOne(req NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	base, err := url.Parse(req.url)
	reply := UrlResponse{
		url:  req.url,
		from: req.from,
		code: 999,
	}
	if err != nil {
		fmt.Println("ERROR: failed to Parse \"" + req.url + "\"")
		reply.err = err
		chFinished <- reply
		return
	}
	switch base.Scheme {
	case "mailto", "irc":
		reply.err = fmt.Errorf("%s on page %s", base.Scheme, req.from)
		reply.code = 900
		chFinished <- reply
		return
	}
	resp, err := http.Get(req.url)
	if err != nil {
		fmt.Println("ERROR: Failed to crawl \"" + req.url + "\"  " + err.Error())
		reply.err = err
		chFinished <- reply
		return
	}
	defer func() {
		// Notify that we're done after this function
		reply.code = resp.StatusCode
		chFinished <- reply
	}()

	loc, err := resp.Location()
	if err == nil && req.url != loc.String() {
		fmt.Printf("\t crawled \"%s\"", req.url)
		fmt.Printf("\t\t to \"%s\"", loc)
	}

	b := resp.Body
	defer b.Close() // close Body when the function returns

	// only parse if this page is on the original site
	// if we moved this check back to the main loop, we'd parse more sites
	if !strings.HasPrefix(req.url, seedUrl) {
		return
	}

	z := html.NewTokenizer(b)

	for {
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			// End of the document, we're done
			return
		case tt == html.StartTagToken:
			t := z.Token()

			// Check if the token is an <a> tag
			isAnchor := t.Data == "a"
			if !isAnchor {
				continue
			}

			// Extract the href value, if there is one
			ok, newUrl := getHref(t)
			if !ok {
				continue
			}
			u, e := url.Parse(newUrl)
			if e != nil {
				fmt.Println("ERROR: failed to Parse \"" + newUrl + "\"")
				continue
			}
			new := NewUrl{
				from: req.url,
				url:  base.ResolveReference(u).String(),
			}
			ch <- new
		}
	}
}

var seedUrl = os.Args[1]

type FoundUrls struct {
	response   int
	usageCount int
	err        error
	from       map[string]int
}

func main() {
	seedUrls := os.Args[1:]

	// Channels
	chUrls := make(chan NewUrl, 1000)
	chWork := make(chan NewUrl, 1000)
	chFinished := make(chan UrlResponse)

	var foundUrls = make(map[string]FoundUrls)

	for w := 1; w <= 10; w++ {
		go crawl(chWork, chUrls, chFinished)
	}

	for _, url := range seedUrls {
		new := NewUrl{
			from: "",
			url:  url,
		}
		chUrls <- new
	}

	// Subscribe to both channels
	count := 0
	for len(chUrls) > 0 || count > 0 {
		select {
		case foundUrl := <-chUrls:
			// don't need to check err - its already been checked before its put in the chUrls que
			u, _ := url.Parse(foundUrl.url)
			// TODO: need a different pipeline for ensuring anchor fragments exist
			// TODO: consider only removing the query/fragment for docs urls
			u.RawQuery = ""
			u.Fragment = ""
			resourceUrl := u.String()

			f, ok := foundUrls[resourceUrl]
			if !ok {
				count++
				f.usageCount = 0
				f.response = 0
				f.from = make(map[string]int)
				f.from[foundUrl.from] = 1
				chWork <- NewUrl{<commit_msg>Give the user some idea what to do<commit_after>package main

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"

	"golang.org/x/net/html"
)

// Helper function to pull the href attribute from a Token
func getHref(t html.Token) (ok bool, href string) {
	// Iterate over all of the Token's attributes until we find an "href"
	for _, a := range t.Attr {
		if a.Key == "href" {
			href = a.Val
			ok = true
		}
	}

	// "bare" return will return the variables (ok, href) as defined in
	// the function definition
	return
}

type UrlResponse struct {
	from string
	url  string
	code int
	err  error
}

type NewUrl struct {
	from string
	url  string
}

func crawl(chWork chan NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	for true {
		new := <-chWork
		crawlOne(new, ch, chFinished)
	}
}

// Extract all http** links from a given webpage
func crawlOne(req NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	base, err := url.Parse(req.url)
	reply := UrlResponse{
		url:  req.url,
		from: req.from,
		code: 999,
	}
	fmt.Printf("Crawling: %s\n", req.url)
	if err != nil {
		fmt.Println("ERROR: failed to Parse \"" + req.url + "\"")
		reply.err = err
		chFinished <- reply
		return
	}
	switch base.Scheme {
	case "mailto", "irc":
		reply.err = fmt.Errorf("%s on page %s", base.Scheme, req.from)
		reply.code = 900
		chFinished <- reply
		return
	}
	resp, err := http.Get(req.url)
	if err != nil {
		fmt.Println("ERROR: Failed to crawl \"" + req.url + "\"  " + err.Error())
		reply.err = err
		chFinished <- reply
		return
	}
	defer func() {
		// Notify that we're done after this function
		reply.code = resp.StatusCode
		chFinished <- reply
	}()

	loc, err := resp.Location()
	if err == nil && req.url != loc.String() {
		fmt.Printf("\t crawled \"%s\"", req.url)
		fmt.Printf("\t\t to \"%s\"", loc)
	}

	b := resp.Body
	defer b.Close() // close Body when the function returns

	// only parse if this page is on the original site
	// if we moved this check back to the main loop, we'd parse more sites
	if !strings.HasPrefix(req.url, seedUrl) {
		return
	}

	z := html.NewTokenizer(b)

	for {
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			// End of the document, we're done
			return
		case tt == html.StartTagToken:
			t := z.Token()

			// Check if the token is an <a> tag
			isAnchor := t.Data == "a"
			if !isAnchor {
				continue
			}

			// Extract the href value, if there is one
			ok, newUrl := getHref(t)
			if !ok {
				continue
			}
			u, e := url.Parse(newUrl)
			if e != nil {
				fmt.Println("ERROR: failed to Parse \"" + newUrl + "\"")
				continue
			}
			new := NewUrl{
				from: req.url,
				url:  base.ResolveReference(u).String(),
			}
			ch <- new
		}
	}
}

var seedUrl string

type FoundUrls struct {
	response   int
	usageCount int
	err        error
	from       map[string]int
}

func main() {
	if len(os.Args[1:]) == 0 {
		fmt.Println("Please specify a URL to check")
		os.Exit(-1)
	}
	seedUrl := os.Args[1]

	// Channels
	chUrls := make(chan NewUrl, 1000)
	chWork := make(chan NewUrl, 1000)
	chFinished := make(chan UrlResponse)

	var foundUrls = make(map[string]FoundUrls)

	for w := 1; w <= 10; w++ {
		go crawl(chWork, chUrls, chFinished)
	}

	new := NewUrl{
		from: "",
		url:  seedUrl,
	}
	chUrls <- new

	// Subscribe to both channels
	count := 0
	for len(chUrls) > 0 || count > 0 {
		select {
		case foundUrl := <-chUrls:
			// don't need to check err - its already been checked before its put in the chUrls que
			u, _ := url.Parse(foundUrl.url)
			// TODO: need a different pipeline for ensuring anchor fragments exist
			// TODO: consider only removing the query/fragment for docs urls
			u.RawQuery = ""
			u.Fragment = ""
			resourceUrl := u.String()

			f, ok := foundUrls[resourceUrl]
			if !ok {
				count++
				f.usageCount = 0
				f.response = 0
				f.from = make(map[string]int)
				f.from[foundUrl.from] = 1
				chWork <- NewUrl{
{'fraction_non_alphanumeric': 0.13665323104927837, 'fraction_numerical': 0.006761149395397218, 'mean_word_length': 2.701636188642926, 'pattern_counts': {'":': 2, '<': 25, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3567668', 'n_tokens_mistral': 3196, 'n_tokens_neox': 2795, 'n_words': 887}