<commit_before>package main

import (
	"net/http"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

func ScrapeWord(word string) []*Entry {
	return Scrape("http://dle.rae.es/srv/search?w="+word, word)
}

func Scrape(url string, word string) []*Entry {
	req, err := http.NewRequest("GET", url, nil)
	req.Header.Set("User-Agent", "")

	client := &http.Client{}
	resp, err := client.Do(req)

	if err != nil {
		panic(err)
	}

	defer resp.Body.Close()
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		panic(err)
	}

	//Look for entries.
	nodes := doc.Find("article")
	//If no entries were found, there is probably a list of links to definitions.
	if nodes.Length() == 0 {
		//Choose the link for the word that is not a verb.
		doc.Find("li").EachWithBreak(func(i int, s *goquery.Selection) bool {
			url, _ = s.Find("a").Attr("href")
			if !strings.HasSuffix(s.Text(), "r.") {
				return false
			}
			if !strings.HasSuffix(s.Text(), "rse.") {
				return false
			}
			return true
		})
		return Scrape("http://dle.rae.es/srv/"+url, word)
	}

	entries := []*Entry{}
	nodes.Each(func(k int, s *goquery.Selection) {
		etymology := s.Find("p.n2").Text()
		defs := []*Definition{}
		vars := []*Variation{}

		s.Find("p[class^='j']").Each(func(i int, s *goquery.Selection) {
			defs = append(defs, ScrapeDefinition(s))
		})

		s.Find("p[class^='k']").Each(func(i int, s *goquery.Selection) {
			vars = append(vars, &Variation{Variation: s.Text()})

			s.NextAll().EachWithBreak(func(_ int, s *goquery.Selection) bool {
				class, _ := s.Attr("class")
				if strings.HasPrefix(class, "l") {
					vars[i].Definitions = append(vars[i].Definitions, &Definition{Definition: s.Text()})
					return false
				}
				if class != "m" {
					return false
				}
				vars[i].Definitions = append(vars[i].Definitions, ScrapeDefinition(s))
				return true<commit_msg>Fix chosing word if there are many<commit_after>package main

import (
	"net/http"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

func ScrapeWord(word string) []*Entry {
	return Scrape("http://dle.rae.es/srv/search?w="+word, word)
}

func Scrape(url string, word string) []*Entry {
	req, err := http.NewRequest("GET", url, nil)
	req.Header.Set("User-Agent", "")

	client := &http.Client{}
	resp, err := client.Do(req)

	if err != nil {
		panic(err)
	}

	defer resp.Body.Close()
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		panic(err)
	}

	//Look for entries.
	nodes := doc.Find("article")
	//If no entries were found, there is probably a list of links to definitions.
	if nodes.Length() == 0 {
		//Choose the link for the word that is not a verb.
		doc.Find("li").EachWithBreak(func(i int, s *goquery.Selection) bool {
			url, _ = s.Find("a").Attr("href")
			t := strings.TrimSpace(s.Text())
			t = t[:len(t)-1]
			if strings.HasSuffix(t, "r") {
				return true
			}
			if strings.HasSuffix(t, "rse") {
				return true
			}
			return false
		})
		return Scrape("http://dle.rae.es/srv/"+url, word)
	}

	entries := []*Entry{}
	nodes.Each(func(k int, s *goquery.Selection) {
		etymology := s.Find("p.n2").Text()
		defs := []*Definition{}
		vars := []*Variation{}

		s.Find("p[class^='j']").Each(func(i int, s *goquery.Selection) {
			defs = append(defs, ScrapeDefinition(s))
		})

		s.Find("p[class^='k']").Each(func(i int, s *goquery.Selection) {
			vars = append(vars, &Variation{Variation: s.Text()})

			s.NextAll().EachWithBreak(func(_ int, s *goquery.Selection) bool {
				class, _ := s.Attr("class")
				if strings.HasPrefix(class, "l") {
					vars[i].Definitions = append(vars[i].Definitions, &Definition{Definition: s.Text()})
					return false
				}
				if class != "m" {
					return false
				}
				vars[i].Definitions = append(vars[i].Definitions, ScrapeDefinition(s))
				return true
{'fraction_non_alphanumeric': 0.18482284505552618, 'fraction_numerical': 0.0013220518244315177, 'mean_word_length': 3.602189781021898, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '438970', 'n_tokens_mistral': 1580, 'n_tokens_neox': 1384, 'n_words': 311}