<commit_before>package unfurl

import (
	"context"
	"io"
	"net/http"
	"strings"
	"time"

	"github.com/keybase/client/go/chat/attachments"

	"github.com/gocolly/colly"
	"github.com/keybase/client/go/protocol/chat1"
)

func fullURL(hostname, path string) string {
	if strings.HasPrefix(path, "http") {
		return path
	} else if strings.HasPrefix(path, "//") {
		return "http:" + path
	} else {
		return "http://" + hostname + path
	}
}

func (s *Scraper) setAndParsePubTime(ctx context.Context, content string, generic *scoredGenericRaw, score int) {
	s.Debug(ctx, "scrapeGeneric: pubdate: %s", content)
	formats := []string{
		"2006-01-02T15:04:05Z",
		"20060102",
	}
	var t time.Time
	var err error
	for _, f := range formats {
		if t, err = time.Parse(f, content); err != nil {
			s.Debug(ctx, "scrapeGeneric: failed to parse pubdate: format: %s err: %s", f, err)
		} else {
			break
		}
	}
	if err != nil {
		s.Debug(ctx, "scrapeGeneric: failed to parse pubdate with any format")
	} else {
		publishTime := int(t.Unix())
		s.Debug(ctx, "scrapeGeneric: success: %d", publishTime)
		generic.setPublishTime(&publishTime, score)
	}
}

func (s *Scraper) setAttr(ctx context.Context, attr, hostname, domain string, generic *scoredGenericRaw, e *colly.HTMLElement) {
	ranker, ok := attrRankMap[attr]
	if !ok { // invalid attribute, ignore
		return
	}
	contents := ranker.content(e)
	score := ranker.score(domain, e)
	for _, content := range contents {
		content = strings.Trim(content, " ")
		if content == "" {
			continue
		}
		switch ranker.setter {
		case setTitle:
			generic.setTitle(content, score)
		case setURL:
			url := fullURL(hostname, content)
			generic.setURL(url, score)
		case setSiteName:
			generic.setSiteName(content, score)
		case setFaviconURL:
			url := fullURL(hostname, content)
			generic.setFaviconURL(&url, score)
		case setImageURL:
			url := fullURL(hostname, content)
			generic.setImageURL(&url, score)
		case setPublishTime:
			s.setAndParsePubTime(ctx, content, generic, score)
		case setDescription:
			generic.setDescription(&content, score)
		}
	}
}

type bodyReadResetter struct {
	io.ReadCloser
}

func (b bodyReadResetter) Reset() error {
	return nil
}

func (s *Scraper) tryAppleTouchIcon(ctx context.Context, generic *scoredGenericRaw, uri, domain string) {
	path, err := GetDefaultAppleTouchURL(uri)
	if err != nil {
		s.Debug(ctx, "tryAppleTouchIcon: failed to get Apple touch URL: %s", err)
		return
	}
	resp, err := http.Get(path)
	if err != nil {
		s.Debug(ctx, "tryAppleTouchIcon: failed to read Apple touch icon: %s", err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 200 && resp.StatusCode <= 299 {
		s.Debug(ctx, "tryAppleTouchIcon: found Apple touch icon at known path")
		mimeType, err := attachments.DetectMIMEType(ctx, bodyReadResetter{ReadCloser: resp.Body},
			"apple-touch-icon.png")
		if err != nil {
			s.Debug(ctx, "tryAppleTouchIcon: failed to get MIME type from response: %s", err)
			return
		}
		if mimeType != "image/png" {
			s.Debug(ctx, "tryAppleTouchIcon: response not a PNG: %s", mimeType)
			return
		}
		generic.setFaviconURL(&path, getAppleTouchFaviconScoreFromPath())
	}
}

func (s *Scraper) scrapeGeneric(ctx context.Context, uri, domain string) (res chat1.UnfurlRaw, err error) {
	hostname, err := GetHostname(uri)
	if err != nil {
		return res, err
	}

	// default favicon location as a fallback
	defaultFaviconURL, err := GetDefaultFaviconURL(uri)
	if err != nil {
		return res, err
	}

	// setup some defaults with score 0 and hope we can find better info.
	generic := new(scoredGenericRaw)
	generic.setURL(uri, 0)
	generic.setSiteName(domain, 0)
	generic.setFaviconURL(&defaultFaviconURL, 0)

	// Run the Colly scraper
	c := colly.NewCollector()
	c.OnHTML("head title", func(e *colly.HTMLElement) {
		s.setAttr(ctx, "title", hostname, domain, generic, e)
	})
	c.OnHTML("head link[rel][href]", func(e *colly.HTMLElement) {
		rel := strings.ToLower(e.Attr("rel"))<commit_msg>improve scraper success by making request look more real CORE-9566 (#14860)<commit_after>package unfurl

import (
	"context"
	"io"
	"net/http"
	"strings"
	"time"

	"github.com/keybase/client/go/chat/attachments"

	"github.com/gocolly/colly"
	"github.com/keybase/client/go/protocol/chat1"
)

func fullURL(hostname, path string) string {
	if strings.HasPrefix(path, "http") {
		return path
	} else if strings.HasPrefix(path, "//") {
		return "http:" + path
	} else {
		return "http://" + hostname + path
	}
}

func (s *Scraper) setAndParsePubTime(ctx context.Context, content string, generic *scoredGenericRaw, score int) {
	s.Debug(ctx, "scrapeGeneric: pubdate: %s", content)
	formats := []string{
		"2006-01-02T15:04:05Z",
		"20060102",
	}
	var t time.Time
	var err error
	for _, f := range formats {
		if t, err = time.Parse(f, content); err != nil {
			s.Debug(ctx, "scrapeGeneric: failed to parse pubdate: format: %s err: %s", f, err)
		} else {
			break
		}
	}
	if err != nil {
		s.Debug(ctx, "scrapeGeneric: failed to parse pubdate with any format")
	} else {
		publishTime := int(t.Unix())
		s.Debug(ctx, "scrapeGeneric: success: %d", publishTime)
		generic.setPublishTime(&publishTime, score)
	}
}

func (s *Scraper) setAttr(ctx context.Context, attr, hostname, domain string, generic *scoredGenericRaw, e *colly.HTMLElement) {
	ranker, ok := attrRankMap[attr]
	if !ok { // invalid attribute, ignore
		return
	}
	contents := ranker.content(e)
	score := ranker.score(domain, e)
	for _, content := range contents {
		content = strings.Trim(content, " ")
		if content == "" {
			continue
		}
		switch ranker.setter {
		case setTitle:
			generic.setTitle(content, score)
		case setURL:
			url := fullURL(hostname, content)
			generic.setURL(url, score)
		case setSiteName:
			generic.setSiteName(content, score)
		case setFaviconURL:
			url := fullURL(hostname, content)
			generic.setFaviconURL(&url, score)
		case setImageURL:
			url := fullURL(hostname, content)
			generic.setImageURL(&url, score)
		case setPublishTime:
			s.setAndParsePubTime(ctx, content, generic, score)
		case setDescription:
			generic.setDescription(&content, score)
		}
	}
}

type bodyReadResetter struct {
	io.ReadCloser
}

func (b bodyReadResetter) Reset() error {
	return nil
}

func (s *Scraper) tryAppleTouchIcon(ctx context.Context, generic *scoredGenericRaw, uri, domain string) {
	path, err := GetDefaultAppleTouchURL(uri)
	if err != nil {
		s.Debug(ctx, "tryAppleTouchIcon: failed to get Apple touch URL: %s", err)
		return
	}
	resp, err := http.Get(path)
	if err != nil {
		s.Debug(ctx, "tryAppleTouchIcon: failed to read Apple touch icon: %s", err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 200 && resp.StatusCode <= 299 {
		s.Debug(ctx, "tryAppleTouchIcon: found Apple touch icon at known path")
		mimeType, err := attachments.DetectMIMEType(ctx, bodyReadResetter{ReadCloser: resp.Body},
			"apple-touch-icon.png")
		if err != nil {
			s.Debug(ctx, "tryAppleTouchIcon: failed to get MIME type from response: %s", err)
			return
		}
		if mimeType != "image/png" {
			s.Debug(ctx, "tryAppleTouchIcon: response not a PNG: %s", mimeType)
			return
		}
		generic.setFaviconURL(&path, getAppleTouchFaviconScoreFromPath())
	}
}

func (s *Scraper) scrapeGeneric(ctx context.Context, uri, domain string) (res chat1.UnfurlRaw, err error) {
	hostname, err := GetHostname(uri)
	if err != nil {
		return res, err
	}

	// default favicon location as a fallback
	defaultFaviconURL, err := GetDefaultFaviconURL(uri)
	if err != nil {
		return res, err
	}

	// setup some defaults with score 0 and hope we can find better info.
	generic := new(scoredGenericRaw)
	generic.setURL(uri, 0)
	generic.setSiteName(domain, 0)
	generic.setFaviconURL(&defaultFaviconURL, 0)

	// Run the Colly scraper
	c := colly.NewCollector(
		colly.UserAgent("Mozilla/5.0 (compatible; Keybase; +https://keybase.io)"),
	)
	c.OnRequest(func(r *colly.Request) {
		r.Headers.Set("connection", "keep-alive")
		r.Headers.Set("upgrade-insecure-requests", "1")
	})
	c.OnHTML("head title", func(e *colly.HTMLElement) {
		s.setAttr(ctx, "title", hostname, domain, generic, e)
	})
	c.OnHTML("head link[rel][href]", func(e *colly.HTMLElement) {
		rel := strings.ToLower(e.Attr("rel"))
{'fraction_non_alphanumeric': 0.1370781040575119, 'fraction_numerical': 0.009747776288534179, 'mean_word_length': 4.238034460753031, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3845695', 'n_tokens_mistral': 3192, 'n_tokens_neox': 2956, 'n_words': 753}