<commit_before>package scanner

import (
	"fmt"
	"net/url"
	"sync"
	"sync/atomic"

	"github.com/afeld/tangle/models"
)

func checkLink(source *url.URL, link models.Link, wg *sync.WaitGroup, numBrokenLinks *uint32) {
	defer wg.Done()

	if !link.IsValid() {
		// https://gobyexample.com/atomic-counters
		atomic.AddUint32(numBrokenLinks, 1)

		dest, _ := link.DestURL()
		fmt.Printf("%s line %d has broken link to %s.\n", source.String(), link.Node.LineNumber(), dest)
	}
}

func scanLinks(source *url.URL, links []models.Link) (numBroken uint32) {
	var wg sync.WaitGroup

	for _, link := range links {
		wg.Add(1)
		go checkLink(source, link, &wg, &numBroken)
	}
	wg.Wait()

	return
}

func ScanPage(source *url.URL) (numBrokenLinks uint32, err error) {
	page := models.Page{AbsURL: source}
	links, err := page.GetLinks()
	if err != nil {
		return
	}

	fmt.Printf("Number of links found: %d\n", len(links))
	numBrokenLinks = scanLinks(source, links)

	return
}<commit_msg>simplify scanner: doesn't need to pass source URL around<commit_after>package scanner

import (
	"fmt"
	"net/url"
	"sync"
	"sync/atomic"

	"github.com/afeld/tangle/models"
)

func checkLink(link models.Link, wg *sync.WaitGroup, numBrokenLinks *uint32) {
	defer wg.Done()

	if !link.IsValid() {
		// https://gobyexample.com/atomic-counters
		atomic.AddUint32(numBrokenLinks, 1)

		dest, _ := link.DestURL()
		fmt.Printf("%s line %d has broken link to %s.\n", link.SourceURL.String(), link.Node.LineNumber(), dest)
	}
}

func ScanLinks(links []models.Link) (numBroken uint32) {
	var wg sync.WaitGroup

	for _, link := range links {
		wg.Add(1)
		go checkLink(link, &wg, &numBroken)
	}
	wg.Wait()

	return
}

func ScanPage(source *url.URL) (numBrokenLinks uint32, err error) {
	page := models.Page{AbsURL: source}
	links, err := page.GetLinks()
	if err != nil {
		return
	}

	fmt.Printf("Number of links found: %d\n", len(links))
	numBrokenLinks = ScanLinks(links)

	return
}
{'fraction_non_alphanumeric': 0.15388574369531652, 'fraction_numerical': 0.010293360782295419, 'mean_word_length': 4.11578947368421, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5749093', 'n_tokens_mistral': 790, 'n_tokens_neox': 739, 'n_words': 174}