<commit_before>package main

import (
  "fmt"
  "log"
  "net/url"
  "errors"

  "github.com/PuerkitoBio/goquery"
)

type ProductUrls struct {
	Url *url.URL
	ImageUrl *url.URL
}

func findProductsOnRandomSearchpage() []*ProductUrls {
	keyword := getRandomSearchKeyword()

	// TODO: This URL only show books :)
	// TODO: Use http://www.amazon.com/s/search-alias%3D[CATEGORY]&field-keywords=[KEYWORDS]
	url := "http://www.amazon.com/s/field-keywords=" + keyword;
	fmt.Println("Finding products on " + url)
	doc, error := goquery.NewDocument(url)
	if error != nil {
		log.Fatal(error)
	}

	return findProductsOnSearchPage(doc)
}

func getRandomSearchKeyword() string {
	return generateRandomSearchString()
}

func findProductsOnSearchPage(doc *goquery.Document) []*ProductUrls {
	var products []*ProductUrls
	doc.Find(".productImage").Each(func (i int, image *goquery.Selection) {
		product, error := extractProduct(image);
		if error == nil {
			products = append(products, &product)
		} else {
			log.Println(error)
		}
	})
	return products
}

func extractProduct(s *goquery.Selection) (ProductUrls, error) {
	parentWithLink := findParentWithHref(s)
	if parentWithLink == nil {
		return ProductUrls{}, errors.New("No parent with href attribute found")
	}

	imageUrl, imageUrlError := attrToUrl(s, "src")
	if imageUrlError == nil {
		link,_ := attrToUrl(parentWithLink, "href")
		return ProductUrls {link, imageUrl}, nil
	}

	return ProductUrls{}, imageUrlError
}

func findParentWithHref(s *goquery.Selection) *goquery.Selection {
	// TODO: Do loop to find first parent with link
	return s.Parent().Parent()
}

func attrToUrl(s *goquery.Selection, attr string) (*url.URL, error) {
	link, exists := s.Attr(attr);
	if exists {
		return url.Parse(link)
	}<commit_msg>Search all search pages for products<commit_after>package main

import (
  "strconv"
  "log"
  "net/url"
  "errors"

  "github.com/PuerkitoBio/goquery"
)

type ProductUrls struct {
	Url *url.URL
	ImageUrl *url.URL
}

func findProductsOnRandomSearchpage() []*ProductUrls {
	page := 0
	keyword := getRandomSearchKeyword()

	var products []*ProductUrls
	shouldContinue := true
	for shouldContinue {
		page = page + 1
		// TODO: This URL only show books :)
		// TODO: Use http://www.amazon.com/s/search-alias%3D[CATEGORY]&field-keywords=[KEYWORDS]
		url := "http://www.amazon.com/s/field-keywords=" + keyword + "&page=" + strconv.Itoa(page);
		newProducts := findProductsOnSearchPageUrl(url)

		products = append(products, newProducts...)
		shouldContinue = len(newProducts) >= 10
	}

	return products
}

func getRandomSearchKeyword() string {
	return generateRandomSearchString()
}

func findProductsOnSearchPageUrl(url string) []*ProductUrls {
	log.Println("Finding products on " + url)
	doc, error := goquery.NewDocument(url)
	if error != nil {
		log.Fatal(error)
	}

	return findProductsOnSearchPage(doc)
}

func findProductsOnSearchPage(doc *goquery.Document) []*ProductUrls {
	var products []*ProductUrls
	doc.Find(".productImage").Each(func (i int, image *goquery.Selection) {
		product, error := extractProduct(image);
		if error == nil {
			products = append(products, &product)
		} else {
			log.Println(error)
		}
	})

	return products
}

func extractProduct(s *goquery.Selection) (ProductUrls, error) {
	parentWithLink := findParentWithHref(s)
	if parentWithLink == nil {
		return ProductUrls{}, errors.New("No parent with href attribute found")
	}

	imageUrl, imageUrlError := attrToUrl(s, "src")
	if imageUrlError == nil {
		link,_ := attrToUrl(parentWithLink, "href")
		return ProductUrls {link, imageUrl}, nil
	}

	return ProductUrls{}, imageUrlError
}

func findParentWithHref(s *goquery.Selection) *goquery.Selection {
	// TODO: Do loop to find first parent with link
	return s.Parent().Parent()
}

func attrToUrl(s *goquery.Selection, attr string) (*url.URL, error) {
	link, exists := s.Attr(attr);
	if exists {
		return url.Parse(link)
	}
{'fraction_non_alphanumeric': 0.13457609805924411, 'fraction_numerical': 0.0015321756894790602, 'mean_word_length': 4.6685962373371925, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4693613', 'n_tokens_mistral': 1391, 'n_tokens_neox': 1317, 'n_words': 313}