<commit_before>package engines

import (
	"fmt"
	"net/url"

	"github.com/PuerkitoBio/goquery"
)

// Google search engine
type Google struct {
	Host, URI string
	SSL       bool
}

// CreateGoogle is the func to create the search engine with default values
func CreateGoogle(ssl bool) Google {
	var schema string
	if ssl {
		schema = "https"
	} else {
		schema = "http"
	}

	return Google{
		schema + "://www.google.com",
		"search?q=site:%s+%s",
		ssl,
	}
}

// GetLinks for a `query` and `site`.
// `site`: What site you want to get links for
// `query`: Search query you want to search on `site` for.
func (google Google) GetLinks(query, site string) (links []string, err error) {
	doc, err := goquery.NewDocument(google.createSearchURL(query, site))
	if err != nil {
		return
	}

	filterLink := func(i int, s *goquery.Selection) {
		href, exists := s.Attr("href")
		if exists {
			links = append(links, google.Host+href)
		}
	}

	doc.Find("a.l").Each(filterLink)
	if len(links) == 0 {
		doc.Find(".r a").Each(filterLink)
	}

	return
}

// createSearchURL is used to create the google search URL.
// Example: `https://www.google.de/search?q=site:stackoverflow.com+windows+get+date+command+line`
func (google Google) createSearchURL(query, site string) string {
	uri := fmt.Sprintf(google.URI, site, url.QueryEscape(query))
	return google.Host + "/" + uri
}<commit_msg>The ssl flag is not used<commit_after>package engines

import (
	"fmt"
	"net/url"

	"github.com/PuerkitoBio/goquery"
)

// Google search engine
type Google struct {
	Host, URI string
}

// CreateGoogle is the func to create the search engine with default values
func CreateGoogle(ssl bool) Google {
	var schema string
	if ssl {
		schema = "https"
	} else {
		schema = "http"
	}

	return Google{
		schema + "://www.google.com",
		"search?q=site:%s+%s",
	}
}

// GetLinks for a `query` and `site`.
// `site`: What site you want to get links for
// `query`: Search query you want to search on `site` for.
func (google Google) GetLinks(query, site string) (links []string, err error) {
	doc, err := goquery.NewDocument(google.createSearchURL(query, site))
	if err != nil {
		return
	}

	filterLink := func(i int, s *goquery.Selection) {
		href, exists := s.Attr("href")
		if exists {
			links = append(links, google.Host+href)
		}
	}

	doc.Find("a.l").Each(filterLink)
	if len(links) == 0 {
		doc.Find(".r a").Each(filterLink)
	}

	return
}

// createSearchURL is used to create the google search URL.
// Example: `https://www.google.com/search?q=site:stackoverflow.com+windows+get+date+command+line`
func (google Google) createSearchURL(query, site string) string {
	uri := fmt.Sprintf(google.URI, site, url.QueryEscape(query))
	return google.Host + "/" + uri
}
{'fraction_non_alphanumeric': 0.1455812247891456, 'fraction_numerical': 0.0007334066740007334, 'mean_word_length': 3.8028169014084505, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '302646', 'n_tokens_mistral': 983, 'n_tokens_neox': 938, 'n_words': 276}