<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package worker

import (
	"github.com/Matir/webborer/logging"
	"github.com/Matir/webborer/util"
	"github.com/Matir/webborer/workqueue"
	"golang.org/x/net/html"
	"io"
	"net/http"
	"net/url"
	"strings"
)

type HTMLWorker struct {
	// Function to add future work
	adder workqueue.QueueAddFunc
}

func NewHTMLWorker(adder workqueue.QueueAddFunc) *HTMLWorker {
	return &HTMLWorker{adder: adder}
}

// Work on this response
func (w *HTMLWorker) Handle(URL *url.URL, body io.Reader) {
	links := w.GetLinks(body)
	foundURLs := make([]*url.URL, 0, len(links))
	for _, l := range links {
		u, err := url.Parse(l)
		if err != nil {
			logging.Logf(logging.LogInfo, "Error parsing URL (%s): %s", l, err.Error())
			continue
		}
		resolved := URL.ResolveReference(u)
		foundURLs = append(foundURLs, resolved)
		// Include parents of the found URL.
		// Worker will remove duplicates
		foundURLs = append(foundURLs, util.GetParentPaths(resolved)...)
	}
	w.adder(foundURLs...)
}

// Check if this response can be handled by this worker
func (*HTMLWorker) Eligible(resp *http.Response) bool {
	ct := resp.Header.Get("Content-type")
	if strings.ToLower(ct) != "text/html" {
		return false
	}
	return resp.ContentLength > 0 && resp.ContentLength < 1024*1024
}

// Get the links for the body.
func (*HTMLWorker) GetLinks(body io.Reader) []string {<commit_msg>Better job of spidering HTML.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package worker

import (
	"github.com/Matir/webborer/logging"
	"github.com/Matir/webborer/util"
	"github.com/Matir/webborer/workqueue"
	"golang.org/x/net/html"
	"io"
	"net/http"
	"net/url"
	"strings"
)

const (
  maxHTMLWorkerSize = 10*1024*1024
)

type HTMLWorker struct {
	// Function to add future work
	adder workqueue.QueueAddFunc
}

func NewHTMLWorker(adder workqueue.QueueAddFunc) *HTMLWorker {
	return &HTMLWorker{adder: adder}
}

// Work on this response
func (w *HTMLWorker) Handle(URL *url.URL, body io.Reader) {
  limitedBody := io.LimitReader(body, maxHTMLWorkerSize)
	links := w.GetLinks(limitedBody)
	foundURLs := make([]*url.URL, 0, len(links))
	for _, l := range links {
		u, err := url.Parse(l)
		if err != nil {
			logging.Logf(logging.LogInfo, "Error parsing URL (%s): %s", l, err.Error())
			continue
		}
		resolved := URL.ResolveReference(u)
		foundURLs = append(foundURLs, resolved)
		// Include parents of the found URL.
		// Worker will remove duplicates
		foundURLs = append(foundURLs, util.GetParentPaths(resolved)...)
	}
	w.adder(foundURLs...)
}

// Check if this response can be handled by this worker
func (*HTMLWorker) Eligible(resp *http.Response) bool {
	ct := resp.Header.Get("Content-type")
	if strings.ToLower(ct) != "text/html" {
		return false
	}
	// ContentLength is often -1, indicating unknown, so we'll try to parse those
	return resp.ContentLength == -1 || (
	  resp.ContentLength > 0 && resp.ContentLength < maxHTMLWorkerSize)
}

// Get the links for the body.
func (*HTMLWorker) GetLinks(body io.Reader) []string {
{'fraction_non_alphanumeric': 0.12425185539861144, 'fraction_numerical': 0.009576250897773522, 'mean_word_length': 3.807825086306099, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '313326', 'n_tokens_mistral': 1392, 'n_tokens_neox': 1316, 'n_words': 438}