<commit_before>package arts

// most of the public interface is defined here

import (
	"bytes"
	"errors"
	"fmt"
	"golang.org/x/net/html"
	"io"
	"regexp"
	"strings"
	//	"fmt"
	"code.google.com/p/go-charset/charset"
	_ "code.google.com/p/go-charset/data"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
)

type Author struct {
	Name    string `json:"name"`
	RelLink string `json:"rellink,omitempty"`
	Email   string `json:"email,omitempty"`
	Twitter string `json:"twitter,omitempty"`
}

type Keyword struct {
	Name string `json:"name"`
	URL  string `json:"url,omitempty"`
}

type Publication struct {
	Name   string `json:"name,omitempty"`
	Domain string `json:"domain,omitempty"`

	// TODO: add publication versions of rel-author
	// eg "article:publisher", rel-publisher
}

type Article struct {
	CanonicalURL string `json:"canonical_url,omitempty"`
	// all known URLs for article (including canonical)
	// TODO: first url should be considered "preferred" if no canonical?
	URLs     []string `json:"urls,omitempty"`
	Headline string   `json:"headline,omitempty"`
	Authors  []Author `json:"authors,omitempty"`
	Content  string   `json:"content,omitempty"`
	// Published contains date of publication.
	// An ISO8601 string is used instead of time.Time, so that
	// less-precise representations can be held (eg YYYY-MM)
	Published   string      `json:"published,omitempty"`
	Updated     string      `json:"updated,omitempty"`
	Publication Publication `json:"publication,omitempty"`
	Keywords    []Keyword   `json:"keywords,omitempty"`
	Section     string      `json:"section,omitempty"`
	// TODO:
	// Language
	// article confidence?
}

func (art *Article) BestURL() string {
	if art.CanonicalURL != "" {
		return art.CanonicalURL
	}
	if len(art.URLs) > 0 {
		return art.URLs[0]
	}
	return ""
}

// TODO:
// - detect non-article pages (index pages etc)

// TODO: pass hints in to the scraper:
// - is it contemporary? (ie not an insanely old or future date)
// - an expected author
// - expected location/timezone
// - expected language

// Debug is the global debug control for the scraper. Set up any loggers you want before calling Extract()
// By default all logging is suppressed.
var Debug = struct {
	// HeadlineLogger is where debug output from the headline extraction will be sent
	HeadlineLogger *log.Logger
	// AuthorsLogger is where debug output from the author extraction will be sent
	AuthorsLogger *log.Logger
	// ContentLogger is where debug output from the content extraction will be sent
	ContentLogger *log.Logger
	// DatesLogger is where debug output from the pubdate/lastupdated extraction will be sent
	DatesLogger *log.Logger

	// URLLoggger is where debug output from URL extraction will be sent (rel-canonical etc)
	URLLogger *log.Logger
}{}

// delete this and leave it up to user?
func Extract(client *http.Client, srcURL string) (*Article, error) {

	resp, err := client.Get(srcURL)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return nil, errors.New(fmt.Sprintf("HTTP error: %s", resp.Status))
	}

	rawHTML, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return ExtractFromHTML(rawHTML, srcURL)

}

func ParseHTML(rawHTML []byte) (*html.Node, error) {
	enc := findCharset("", rawHTML)
	var r io.Reader
	r = strings.NewReader(string(rawHTML))
	if enc != "utf-8" {
		// we'll be translating to utf-8
		var err error
		r, err = charset.NewReader(enc, r)
		if err != nil {
			return nil, err
		}
	}

	return html.Parse(r)
}

func ExtractFromHTML(rawHTML []byte, artURL string) (*Article, error) {

	root, err := ParseHTML(rawHTML)
	if err != nil {
		return nil, err
	}

	return ExtractFromTree(root, artURL)
}

func ExtractFromTree(root *html.Node, artURL string) (*Article, error) {

	art := &Article{}

	// fill in any missing loggers to discard
	if Debug.HeadlineLogger == nil {
		Debug.HeadlineLogger = log.New(ioutil.Discard, "", 0)
	}
	if Debug.AuthorsLogger == nil {
		Debug.AuthorsLogger = log.New(ioutil.Discard, "", 0)
	}
	if Debug.ContentLogger == nil {
		Debug.ContentLogger = log.New(ioutil.Discard, "", 0)
	}
	if Debug.DatesLogger == nil {
		Debug.DatesLogger = log.New(ioutil.Discard, "", 0)
	}
	if Debug.URLLogger == nil {
		Debug.URLLogger = log.New(ioutil.Discard, "", 0)
	}

	//	html.Render(dbug, root)
	u, err := url.Parse(artURL)
	if err != nil {
		return nil, err
	}

	art.Section = grabSection(root, u)

	// TODO: do we really need to do this?<commit_msg>provide null debugger loggers by default<commit_after>package arts

// most of the public interface is defined here

import (
	"bytes"
	"errors"
	"fmt"
	"golang.org/x/net/html"
	"io"
	"regexp"
	"strings"
	//	"fmt"
	"code.google.com/p/go-charset/charset"
	_ "code.google.com/p/go-charset/data"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
)

type Author struct {
	Name    string `json:"name"`
	RelLink string `json:"rellink,omitempty"`
	Email   string `json:"email,omitempty"`
	Twitter string `json:"twitter,omitempty"`
}

type Keyword struct {
	Name string `json:"name"`
	URL  string `json:"url,omitempty"`
}

type Publication struct {
	Name   string `json:"name,omitempty"`
	Domain string `json:"domain,omitempty"`

	// TODO: add publication versions of rel-author
	// eg "article:publisher", rel-publisher
}

type Article struct {
	CanonicalURL string `json:"canonical_url,omitempty"`
	// all known URLs for article (including canonical)
	// TODO: first url should be considered "preferred" if no canonical?
	URLs     []string `json:"urls,omitempty"`
	Headline string   `json:"headline,omitempty"`
	Authors  []Author `json:"authors,omitempty"`
	Content  string   `json:"content,omitempty"`
	// Published contains date of publication.
	// An ISO8601 string is used instead of time.Time, so that
	// less-precise representations can be held (eg YYYY-MM)
	Published   string      `json:"published,omitempty"`
	Updated     string      `json:"updated,omitempty"`
	Publication Publication `json:"publication,omitempty"`
	Keywords    []Keyword   `json:"keywords,omitempty"`
	Section     string      `json:"section,omitempty"`
	// TODO:
	// Language
	// article confidence?
}

func (art *Article) BestURL() string {
	if art.CanonicalURL != "" {
		return art.CanonicalURL
	}
	if len(art.URLs) > 0 {
		return art.URLs[0]
	}
	return ""
}

// TODO:
// - detect non-article pages (index pages etc)

// TODO: pass hints in to the scraper:
// - is it contemporary? (ie not an insanely old or future date)
// - an expected author
// - expected location/timezone
// - expected language

var nullLogger = log.New(ioutil.Discard, "", 0)

// Debug is the global debug control for the scraper. Set up any loggers you want before calling Extract()
// By default all logging is suppressed.
var Debug = struct {
	// HeadlineLogger is where debug output from the headline extraction will be sent
	HeadlineLogger *log.Logger
	// AuthorsLogger is where debug output from the author extraction will be sent
	AuthorsLogger *log.Logger
	// ContentLogger is where debug output from the content extraction will be sent
	ContentLogger *log.Logger
	// DatesLogger is where debug output from the pubdate/lastupdated extraction will be sent
	DatesLogger *log.Logger

	// URLLoggger is where debug output from URL extraction will be sent (rel-canonical etc)
	URLLogger *log.Logger
}{
	nullLogger,
	nullLogger,
	nullLogger,
	nullLogger,
	nullLogger,
}

// delete this and leave it up to user?
func Extract(client *http.Client, srcURL string) (*Article, error) {

	resp, err := client.Get(srcURL)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return nil, errors.New(fmt.Sprintf("HTTP error: %s", resp.Status))
	}

	rawHTML, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return ExtractFromHTML(rawHTML, srcURL)

}

func ParseHTML(rawHTML []byte) (*html.Node, error) {
	enc := findCharset("", rawHTML)
	var r io.Reader
	r = strings.NewReader(string(rawHTML))
	if enc != "utf-8" {
		// we'll be translating to utf-8
		var err error
		r, err = charset.NewReader(enc, r)
		if err != nil {
			return nil, err
		}
	}

	return html.Parse(r)
}

func ExtractFromHTML(rawHTML []byte, artURL string) (*Article, error) {

	root, err := ParseHTML(rawHTML)
	if err != nil {
		return nil, err
	}

	return ExtractFromTree(root, artURL)
}

func ExtractFromTree(root *html.Node, artURL string) (*Article, error) {

	art := &Article{}

	//	html.Render(dbug, root)
	u, err := url.Parse(artURL)
	if err != nil {
		return nil, err
	}

	art.Section = grabSection(root, u)

	// TODO: do we really need to do this?
{'fraction_non_alphanumeric': 0.12388042340351285, 'fraction_numerical': 0.0032569500988716992, 'mean_word_length': 3.8006700167504186, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2426197', 'n_tokens_mistral': 2941, 'n_tokens_neox': 2796, 'n_words': 892}