<commit_before>package middleware

import (
	"net/http"

	"github.com/PuerkitoBio/goquery"
)

// ExtractAllLinks simply extracts all the <a href="<urL>"> </a> in a page
// provided that they aren't silly like a hash. It also expands relative
// links automagically.
func ExtractAllLinks(resp *http.Response) []string {
	var links []string

	tempURL := *resp.Request.URL
	tempURL.Fragment = ""
	tempURL.Path = ""
	tempURL.RawQuery = ""
	uri := tempURL.String()

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	resp.Body.Close()
	if err != nil {
		return links
	}

	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		href, exists := s.Attr("href")

		if exists && href != "" {

			if href[0] == '#' {
				return
			}
			if href[0] == '/' {
				href = uri + href
			}
			links = append(links, href)
		}
	})

	return links
}<commit_msg>Added double slash href handling to ExtractAllLinks Parse function.<commit_after>package middleware

import (
	"net/http"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

// ExtractAllLinks simply extracts all the <a href="<urL>"> </a> in a page
// provided that they aren't silly like a hash. It also expands relative
// links automagically.
func ExtractAllLinks(resp *http.Response) []string {
	var links []string

	tempURL := *resp.Request.URL
	tempURL.Fragment = ""
	tempURL.Path = ""
	tempURL.RawQuery = ""
	uri := tempURL.String()

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	resp.Body.Close()
	if err != nil {
		return links
	}

	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		href, exists := s.Attr("href")

		if exists && href != "" {

			if href[0] == '#' {
				return
			}
			if strings.HasPrefix(href, "//") {
				href = "http:" + href
			}
			if strings.HasPrefix(href, "/") {
				href = uri + href
			}
			links = append(links, href)
		}
	})

	return links
}
{'fraction_non_alphanumeric': 0.15560391730141457, 'fraction_numerical': 0.001632208922742111, 'mean_word_length': 3.2867132867132867, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6095106', 'n_tokens_mistral': 715, 'n_tokens_neox': 636, 'n_words': 166}