<commit_before>package hermes

import (
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"sync"

	"errors"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
)

// Scrape function will take a url and fire off pipelines to scrape titles,
// paragraphs, divs and return a Document struct with valid title, content and a link
func Scrape(u string, cs CustomSettings) (Document, error) {
	document := Document{}
	for document = range documentGenerator(rootGenerator(respGenerator(u)), cs) {
		return document, nil
	}
	return document, errors.New("Scraping error")
}

// function to generate a response from a url pass into it
func respGenerator(url string) <-chan *http.Response {
	var wg sync.WaitGroup
	out := make(chan *http.Response)
	wg.Add(1)
	go func(url string) {
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			panic(err)
		}
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			panic(err)
		}
		out <- resp
		wg.Done()
	}(url)
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

// function to generate an html node with an http.Response pointer passed into it
func rootGenerator(in <-chan *http.Response) <-chan *html.Node {
	var wg sync.WaitGroup
	out := make(chan *html.Node)
	for resp := range in {
		wg.Add(1)
		go func(resp *http.Response) {
			root, err := html.Parse(resp.Body)
			if err != nil {
				panic(err)
			}
			out <- root
			wg.Done()
		}(resp)
	}
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

// documentGenerator function will take in a channel with a pointer to an html.Node
// type and customized settings and it will fire off scraping mechanisms to return a Document
func documentGenerator(in <-chan *html.Node, cs CustomSettings) <-chan Document {
	var wg sync.WaitGroup
	out := make(chan Document)
	for root := range in {
		wg.Add(1)
		go func(root *html.Node) {
			doc := goquery.NewDocumentFromNode(root)
			out <- scrapeDocument(cs.RootLink, doc, cs.Tags)
			wg.Done()
		}(root)
	}
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

// function to scrape a goquery document and return a structured Document back
func scrapeDocument(u string, doc *goquery.Document, tags []string) Document {
	var (
		d       Document
		content string
	)
	// scrape page <Title>
	doc.Find("head").Each(func(i int, s *goquery.Selection) {
		title := s.Find("title").Text()
		d.Title = strings.TrimSpace(strings.Replace(title, " ", " ", -1))
	})

	// scrape page <Description>
	doc.Find("meta").Each(func(i int, s *goquery.Selection) {
		if name, _ := s.Attr("name"); strings.EqualFold(name, "description") {
			description, _ := s.Attr("content")
			d.Description = strings.TrimSpace(strings.Replace(description, " ", " ", -1))
		}
	})

	if len(tags) > 0 {
		for _, tag := range tags {
			text := returnText(doc, tag)
			content += " " + strings.TrimSpace(strings.Replace(text, " ", " ", -1))
		}
	} else {
		text := returnText(doc, "default")
		content += " " + strings.TrimSpace(strings.Replace(text, " ", " ", -1))
	}

	val, err := url.Parse(doc.Url.String())
	if err != nil {
		fmt.Printf("error: resolve URL %s - %s\n", val, err)
	}

	d.Tag = generateTag(val.Host)

	d.Content = content
	d.Link = val.String()

	return d
}

// function to take a custom tag or "default" and return text from that in the goquery document
func returnText(doc *goquery.Document, tag string) (text string) {
	doc.Find("body").Each(func(i int, s *goquery.Selection) {
		// default to pulling all the div and p tags else pull custom setting tags
		if tag == "default" {
			text += " " + s.Find("p").Text()
			text += " " + s.Find("div").Text()
		} else {
			text += " " + s.Find(tag).Text()
		}
	})
	return
}

// generate a tag for a link/document based on the first url string
// (>>sub<< in sub.domain.com or >>domain<< in domain.com)
func generateTag(u string) (tag string) {
	s := strings.Split(u, ".")
	if s[0] == "www" && len(s) > 0 {
		tag = s[1]
	} else {
		tag = s[0]
	}
	return
}<commit_msg>another small fix for document links<commit_after>package hermes

import (
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"sync"

	"errors"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
)

// Scrape function will take a url and fire off pipelines to scrape titles,
// paragraphs, divs and return a Document struct with valid title, content and a link
func Scrape(u string, cs CustomSettings) (Document, error) {
	document := Document{}
	for document = range documentGenerator(rootGenerator(respGenerator(u)), u, cs) {
		return document, nil
	}
	return document, errors.New("Scraping error")
}

// function to generate a response from a url pass into it
func respGenerator(url string) <-chan *http.Response {
	var wg sync.WaitGroup
	out := make(chan *http.Response)
	wg.Add(1)
	go func(url string) {
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			panic(err)
		}
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			panic(err)
		}
		out <- resp
		wg.Done()
	}(url)
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

// function to generate an html node with an http.Response pointer passed into it
func rootGenerator(in <-chan *http.Response) <-chan *html.Node {
	var wg sync.WaitGroup
	out := make(chan *html.Node)
	for resp := range in {
		wg.Add(1)
		go func(resp *http.Response) {
			root, err := html.Parse(resp.Body)
			if err != nil {
				panic(err)
			}
			out <- root
			wg.Done()
		}(resp)
	}
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

// documentGenerator function will take in a channel with a pointer to an html.Node
// type and customized settings and it will fire off scraping mechanisms to return a Document
func documentGenerator(in <-chan *html.Node, link string, cs CustomSettings) <-chan Document {
	var wg sync.WaitGroup
	out := make(chan Document)
	for root := range in {
		wg.Add(1)
		go func(root *html.Node) {
			doc := goquery.NewDocumentFromNode(root)
			out <- scrapeDocument(link, doc, cs.Tags)
			wg.Done()
		}(root)
	}
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

// function to scrape a goquery document and return a structured Document back
func scrapeDocument(u string, doc *goquery.Document, tags []string) Document {
	var (
		d       Document
		content string
	)
	// scrape page <Title>
	doc.Find("head").Each(func(i int, s *goquery.Selection) {
		title := s.Find("title").Text()
		d.Title = strings.TrimSpace(strings.Replace(title, " ", " ", -1))
	})

	// scrape page <Description>
	doc.Find("meta").Each(func(i int, s *goquery.Selection) {
		if name, _ := s.Attr("name"); strings.EqualFold(name, "description") {
			description, _ := s.Attr("content")
			d.Description = strings.TrimSpace(strings.Replace(description, " ", " ", -1))
		}
	})

	if len(tags) > 0 {
		for _, tag := range tags {
			text := returnText(doc, tag)
			content += " " + strings.TrimSpace(strings.Replace(text, " ", " ", -1))
		}
	} else {
		text := returnText(doc, "default")
		content += " " + strings.TrimSpace(strings.Replace(text, " ", " ", -1))
	}

	val, err := url.Parse(u)
	if err != nil {
		fmt.Printf("error: resolve URL %s - %s\n", val, err)
	}

	d.Tag = generateTag(val.Host)

	d.Content = content
	d.Link = u

	return d
}

// function to take a custom tag or "default" and return text from that in the goquery document
func returnText(doc *goquery.Document, tag string) (text string) {
	doc.Find("body").Each(func(i int, s *goquery.Selection) {
		// default to pulling all the div and p tags else pull custom setting tags
		if tag == "default" {
			text += " " + s.Find("p").Text()
			text += " " + s.Find("div").Text()
		} else {
			text += " " + s.Find(tag).Text()
		}
	})
	return
}

// generate a tag for a link/document based on the first url string
// (>>sub<< in sub.domain.com or >>domain<< in domain.com)
func generateTag(u string) (tag string) {
	s := strings.Split(u, ".")
	if s[0] == "www" && len(s) > 0 {
		tag = s[1]
	} else {
		tag = s[0]
	}
	return
}
{'fraction_non_alphanumeric': 0.15437089439110663, 'fraction_numerical': 0.003031834259727135, 'mean_word_length': 3.3667953667953667, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2292934', 'n_tokens_mistral': 2992, 'n_tokens_neox': 2747, 'n_words': 828}