<commit_before>package scraper

import (
	"bytes"
	"net/url"
	"os"
	"regexp"

	"github.com/cornelk/goscrape/appcontext"

	"github.com/headzoo/surf"
	"github.com/headzoo/surf/agent"
	"github.com/headzoo/surf/browser"
	"github.com/uber-go/zap"
)

type (
	// Scraper contains all scraping data
	Scraper struct {
		ImageQuality uint
		MaxDepth     uint
		URL          *url.URL

		browser  *browser.Browser
		excludes []*regexp.Regexp
		log      zap.Logger

		assets         map[string]bool
		assetsExternal map[string]bool
		pages          map[string]bool
	}
)

// New creates a new Scraper instance
func New(URL string) (*Scraper, error) {
	u, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}

	b := surf.NewBrowser()
	b.SetUserAgent(agent.GoogleBot())

	s := &Scraper{
		browser:        b,
		log:            appcontext.Logger,
		assets:         make(map[string]bool),
		assetsExternal: make(map[string]bool),
		pages:          make(map[string]bool),
		URL:            u,
	}
	return s, nil
}

// SetExcludes sets and checks the exclusions regular expressions
func (s *Scraper) SetExcludes(excludes []string) error {
	for _, e := range excludes {
		re, err := regexp.Compile(e)
		if err != nil {
			return err
		}

		s.excludes = append(s.excludes, re)
		s.log.Debug("Excluding", zap.Stringer("RE", re))
	}

	return nil
}

// Start starts the scraping
func (s *Scraper) Start() error {
	p := s.URL.Path
	if p == "" {
		p = "/"
	}
	s.pages[p] = false
	return s.scrapeURL(s.URL, 0)
}

func (s *Scraper) scrapeURL(URL *url.URL, currentDepth uint) error {
	s.log.Info("Downloading", zap.Stringer("URL", URL))
	err := s.browser.Open(URL.String())
	if err != nil {
		return err
	}

	buf := &bytes.Buffer{}
	_, err = s.browser.Download(buf)
	if err != nil {
		return err
	}

	html, err := s.fixFileReferences(URL, buf)
	if err != nil {
		return err
	}

	buf = bytes.NewBufferString(html)
	filePath := s.GetFilePath(URL, true)
	err = s.writeFile(filePath, buf) // always update html files, content might have changed
	if err != nil {
		return err
	}

	for _, stylesheet := range s.browser.Stylesheets() {
		err = s.downloadAssetURL(&stylesheet.DownloadableAsset)
		if err != nil {
			return nil
		}
	}
	for _, script := range s.browser.Scripts() {
		err = s.downloadAssetURL(&script.DownloadableAsset)<commit_msg>Handle missing URL schemes and site redirections<commit_after>package scraper

import (
	"bytes"
	"net/url"
	"os"
	"regexp"

	"github.com/cornelk/goscrape/appcontext"

	"github.com/headzoo/surf"
	"github.com/headzoo/surf/agent"
	"github.com/headzoo/surf/browser"
	"github.com/uber-go/zap"
)

type (
	// Scraper contains all scraping data
	Scraper struct {
		ImageQuality uint
		MaxDepth     uint
		URL          *url.URL

		browser  *browser.Browser
		excludes []*regexp.Regexp
		log      zap.Logger

		assets         map[string]bool
		assetsExternal map[string]bool
		pages          map[string]bool
	}
)

// New creates a new Scraper instance
func New(URL string) (*Scraper, error) {
	u, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}

	if u.Scheme == "" {
		u.Scheme = "http" // if no URL scheme was given default to http
	}

	b := surf.NewBrowser()
	b.SetUserAgent(agent.GoogleBot())

	s := &Scraper{
		browser:        b,
		log:            appcontext.Logger,
		assets:         make(map[string]bool),
		assetsExternal: make(map[string]bool),
		pages:          make(map[string]bool),
		URL:            u,
	}
	return s, nil
}

// SetExcludes sets and checks the exclusions regular expressions
func (s *Scraper) SetExcludes(excludes []string) error {
	for _, e := range excludes {
		re, err := regexp.Compile(e)
		if err != nil {
			return err
		}

		s.excludes = append(s.excludes, re)
		s.log.Debug("Excluding", zap.Stringer("RE", re))
	}

	return nil
}

// Start starts the scraping
func (s *Scraper) Start() error {
	p := s.URL.Path
	if p == "" {
		p = "/"
	}
	s.pages[p] = false
	return s.scrapeURL(s.URL, 0)
}

func (s *Scraper) scrapeURL(URL *url.URL, currentDepth uint) error {
	s.log.Info("Downloading", zap.Stringer("URL", URL))
	err := s.browser.Open(URL.String())
	if err != nil {
		return err
	}

	buf := &bytes.Buffer{}
	_, err = s.browser.Download(buf)
	if err != nil {
		return err
	}

	if currentDepth == 0 {
		URL = s.browser.Url() // use the URL that the website returned as new base url for the scrape, in case of a redirect
		s.URL = URL
	}

	html, err := s.fixFileReferences(URL, buf)
	if err != nil {
		return err
	}

	buf = bytes.NewBufferString(html)
	filePath := s.GetFilePath(URL, true)
	err = s.writeFile(filePath, buf) // always update html files, content might have changed
	if err != nil {
		return err
	}

	for _, stylesheet := range s.browser.Stylesheets() {
		err = s.downloadAssetURL(&stylesheet.DownloadableAsset)
		if err != nil {
			return nil
		}
	}
	for _, script := range s.browser.Scripts() {
		err = s.downloadAssetURL(&script.DownloadableAsset)
{'fraction_non_alphanumeric': 0.13622480146609653, 'fraction_numerical': 0.0006108735491753207, 'mean_word_length': 3.069594034797017, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6111573', 'n_tokens_mistral': 1916, 'n_tokens_neox': 1784, 'n_words': 454}