<commit_before>// Package recon scrapes URLs for OpenGraph information.
package recon

import (
	"fmt"
	"golang.org/x/net/html"
	"io"
	"math"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"sort"
	"strings"
	"time"

	"image/gif"
	"image/jpeg"
	"image/png"
)

// Parser is the client object and holds the relevant information needed when parsing a URL
type Parser struct {
	client   *http.Client
	req      *http.Request
	metaTags []metaTag
	imgTags  []imgTag
}

// ParseResult is what comes back from a Parse
type ParseResult struct {
	// URL is either the URL as-passed or the defined URL (via og:url) if present
	URL string `json:"url"`

	// Host is the domain of the URL as-passed or the defined URL if present
	Host string `json:"host"`

	// Site is the name of the site as defined via og:site_name or site_name
	Site string `json:"site_name"`

	// Title is the title of the page as defined via og:title or title
	Title string `json:"title"`

	// Type is the type of the page (article, video, etc.) as defined via og:type or type.
	Type string `json:"type"`

	// Description is the description of the page as defined via og:description or description.
	Description string `json:"description"`

	// Author is the author of the page as defined via og:author or author.
	Author string `json:"author"`

	// Publisher is the publisher of the page as defined via og:publisher or publisher.
	Publisher string `json:"publisher"`

	// Images is the collection of images parsed from the page using either og:image meta tags or <img> tags.
	Images []Image `json:"images"`

	// Scraped is the time when the page was scraped (or the time Parse was run).
	Scraped time.Time `json:"scraped"`
}

type metaTag struct {
	name     string
	value    string
	priority float64
}

type imgTag struct {
	url       string
	alt       string
	preferred bool
}

// Image contains information about parsed images on the page
type Image struct {
	URL         string  `json:"url"`
	Type        string  `json:"type"`
	Width       int     `json:"width"`
	Height      int     `json:"height"`
	Alt         string  `json:"alt"`
	AspectRatio float64 `json:"aspectRatio"`
	Preferred   bool    `json:"preferred,omitempty"`
}

type byPreferred []Image

var targetedProperties = map[string]float64{
	"og:site_name":   1,
	"og:title":       1,
	"og:type":        1,
	"og:description": 1,
	"og:author":      1,
	"og:publisher":   1,
	"og:url":         1,
	"og:image":       1,

	"site_name":   0.5,
	"title":       0.5,
	"type":        0.5,
	"description": 0.5,
	"author":      0.5,
	"publisher":   0.5,
}

var propertyMap = map[string][]string{
	"URL":         {"og:url"},
	"Site":        {"og:site_name", "site_name"},
	"Title":       {"og:title", "title"},
	"Type":        {"og:type", "type"},
	"Description": {"og:description", "description"},
	"Author":      {"og:author", "author"},
	"Publisher":   {"og:publisher", "publisher"},
}

// OptimalAspectRatio is the target aspect ratio that recon favors when looking at images
var OptimalAspectRatio = 1.91

// ImageLookupTimeout is the maximum amount of time recon will spend downloading and analyzing images
var ImageLookupTimeout = 10 * time.Second

// NewParser returns a new Parser object
func NewParser() Parser {
	p := Parser{}
	return p
}

func (p *Parser) reset() {
	client := http.DefaultClient
	client.Jar, _ = cookiejar.New(nil)

	p.client = client
	p.imgTags = make([]imgTag, 0)
	p.metaTags = make([]metaTag, 0)
}

// Parse takes a url and attempts to parse it using a default minimum confidence of 0 (all information is used).
func (p *Parser) Parse(url string) (ParseResult, error) {
	return p.ParseWithConfidence(url, 0)
}

// ParseWithConfidence takes a url and attempts to parse it, only considering information with a confidence above the minimum confidence provided (should be between 0 and 1).
func (p *Parser) ParseWithConfidence(url string, confidence float64) (ParseResult, error) {
	if confidence < 0 || confidence > 1 {
		return ParseResult{}, fmt.Errorf("ParseWithConfidence: confidence should between 0 and 1 inclusive, is %v", confidence)
	}

	p.reset()
	resp, err := p.doRequest(url)
	if err != nil {
		return ParseResult{}, err
	}

	p.tokenize(resp.Body, confidence)

	res := p.buildResult()

	return res, nil
}

func (p *Parser) doRequest(url string) (*http.Response, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, fmt.Errorf("error creating request: %s, url: %s", err, url)
	}

	resp, err := p.client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("http error: %s, url: %s", err, url)
	}
<commit_msg>Adding a user agent string to hopefully get past at least rudimentary bot sniffing<commit_after>// Package recon scrapes URLs for OpenGraph information.
package recon

import (
	"fmt"
	"golang.org/x/net/html"
	"io"
	"math"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"sort"
	"strings"
	"time"

	"image/gif"
	"image/jpeg"
	"image/png"
)

// Parser is the client object and holds the relevant information needed when parsing a URL
type Parser struct {
	client   *http.Client
	req      *http.Request
	metaTags []metaTag
	imgTags  []imgTag
}

// ParseResult is what comes back from a Parse
type ParseResult struct {
	// URL is either the URL as-passed or the defined URL (via og:url) if present
	URL string `json:"url"`

	// Host is the domain of the URL as-passed or the defined URL if present
	Host string `json:"host"`

	// Site is the name of the site as defined via og:site_name or site_name
	Site string `json:"site_name"`

	// Title is the title of the page as defined via og:title or title
	Title string `json:"title"`

	// Type is the type of the page (article, video, etc.) as defined via og:type or type.
	Type string `json:"type"`

	// Description is the description of the page as defined via og:description or description.
	Description string `json:"description"`

	// Author is the author of the page as defined via og:author or author.
	Author string `json:"author"`

	// Publisher is the publisher of the page as defined via og:publisher or publisher.
	Publisher string `json:"publisher"`

	// Images is the collection of images parsed from the page using either og:image meta tags or <img> tags.
	Images []Image `json:"images"`

	// Scraped is the time when the page was scraped (or the time Parse was run).
	Scraped time.Time `json:"scraped"`
}

type metaTag struct {
	name     string
	value    string
	priority float64
}

type imgTag struct {
	url       string
	alt       string
	preferred bool
}

// Image contains information about parsed images on the page
type Image struct {
	URL         string  `json:"url"`
	Type        string  `json:"type"`
	Width       int     `json:"width"`
	Height      int     `json:"height"`
	Alt         string  `json:"alt"`
	AspectRatio float64 `json:"aspectRatio"`
	Preferred   bool    `json:"preferred,omitempty"`
}

type byPreferred []Image

var targetedProperties = map[string]float64{
	"og:site_name":   1,
	"og:title":       1,
	"og:type":        1,
	"og:description": 1,
	"og:author":      1,
	"og:publisher":   1,
	"og:url":         1,
	"og:image":       1,

	"site_name":   0.5,
	"title":       0.5,
	"type":        0.5,
	"description": 0.5,
	"author":      0.5,
	"publisher":   0.5,
}

var propertyMap = map[string][]string{
	"URL":         {"og:url"},
	"Site":        {"og:site_name", "site_name"},
	"Title":       {"og:title", "title"},
	"Type":        {"og:type", "type"},
	"Description": {"og:description", "description"},
	"Author":      {"og:author", "author"},
	"Publisher":   {"og:publisher", "publisher"},
}

// OptimalAspectRatio is the target aspect ratio that recon favors when looking at images
var OptimalAspectRatio = 1.91

// ImageLookupTimeout is the maximum amount of time recon will spend downloading and analyzing images
var ImageLookupTimeout = 10 * time.Second

// NewParser returns a new Parser object
func NewParser() Parser {
	p := Parser{}
	return p
}

func (p *Parser) reset() {
	client := http.DefaultClient
	client.Jar, _ = cookiejar.New(nil)

	p.client = client
	p.imgTags = make([]imgTag, 0)
	p.metaTags = make([]metaTag, 0)
}

// Parse takes a url and attempts to parse it using a default minimum confidence of 0 (all information is used).
func (p *Parser) Parse(url string) (ParseResult, error) {
	return p.ParseWithConfidence(url, 0)
}

// ParseWithConfidence takes a url and attempts to parse it, only considering information with a confidence above the minimum confidence provided (should be between 0 and 1).
func (p *Parser) ParseWithConfidence(url string, confidence float64) (ParseResult, error) {
	if confidence < 0 || confidence > 1 {
		return ParseResult{}, fmt.Errorf("ParseWithConfidence: confidence should between 0 and 1 inclusive, is %v", confidence)
	}

	p.reset()
	resp, err := p.doRequest(url)
	if err != nil {
		return ParseResult{}, err
	}

	p.tokenize(resp.Body, confidence)

	res := p.buildResult()

	return res, nil
}

func (p *Parser) doRequest(url string) (*http.Response, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, fmt.Errorf("error creating request: %s, url: %s", err, url)
	}

	req.Header.Add("User-Agent", "recon (github.com/jimmysawczuk/recon; similar to Facebot, facebookexternalhit/1.1)")

	resp, err := p.client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("http error: %s, url: %s", err, url)
	}

{'fraction_non_alphanumeric': 0.12259512612227448, 'fraction_numerical': 0.00940572894399316, 'mean_word_length': 3.557720409157331, 'pattern_counts': {'":': 42, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7495713', 'n_tokens_mistral': 3103, 'n_tokens_neox': 2985, 'n_words': 1108}