<commit_before>package htmlquery

import (
	"bytes"
	"fmt"
	"net/http"
	"strings"

	"github.com/antchfx/gxpath"
	"github.com/antchfx/gxpath/xpath"
	"golang.org/x/net/html"
	"golang.org/x/text/encoding/htmlindex"
)

// CreateXPathNavigator creates a new xpath.NodeNavigator for the specified html.Node.
func CreateXPathNavigator(top *html.Node) xpath.NodeNavigator {
	return &htmlNodeNavigator{curr: top, root: top, attr: -1}
}

// Find searches the html.Node that matches by the specified XPath expr.
func Find(top *html.Node, expr string) []*html.Node {
	var elems []*html.Node
	t := gxpath.Select(CreateXPathNavigator(top), expr)
	for t.MoveNext() {
		elems = append(elems, (t.Current().(*htmlNodeNavigator)).curr)
	}
	return elems
}

// FindOne searches the html.Node that matches by the specified XPath expr,
// and returns first element of matched html.Node.
func FindOne(top *html.Node, expr string) *html.Node {
	var elem *html.Node
	t := gxpath.Select(CreateXPathNavigator(top), expr)
	if t.MoveNext() {
		elem = (t.Current().(*htmlNodeNavigator)).curr
	}
	return elem
}

// FindEach searches the html.Node and calls functions cb.
func FindEach(top *html.Node, expr string, cb func(int, *html.Node)) {
	t := gxpath.Select(CreateXPathNavigator(top), expr)
	i := 0
	for t.MoveNext() {
		cb(i, (t.Current().(*htmlNodeNavigator)).curr)
		i++
	}
}

func Load(url string) (*html.Node, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	charset := "utf-8"
	contyp := strings.ToLower(resp.Header.Get("Content-Type"))
	if i := strings.Index(contyp, "charset="); i > 0 {
		charset = contyp[i+8:]
	}
	e, err := htmlindex.Get(charset)
	if err != nil {
		return nil, err
	}
	if name, _ := htmlindex.Name(e); name != "utf8" {
		body := e.NewDecoder().Reader(resp.Body)
		return html.Parse(body)
	}
	return html.Parse(resp.Body)
}

// InnerText returns the text between the start and end tags of the object.
func InnerText(n *html.Node) string {
	if n.Type == html.TextNode || n.Type == html.CommentNode {
		return n.Data
	}
	var buf bytes.Buffer
	for child := n.FirstChild; child != nil; child = child.NextSibling {
		buf.WriteString(InnerText(child))
	}
	return buf.String()
}

func isSelfClosingTag(t string) bool {
	switch t {<commit_msg>refactor: remove white space for InnerText()<commit_after>package htmlquery

import (
	"bytes"
	"fmt"
	"net/http"
	"strings"

	"github.com/antchfx/gxpath"
	"github.com/antchfx/gxpath/xpath"
	"golang.org/x/net/html"
	"golang.org/x/text/encoding/htmlindex"
)

// CreateXPathNavigator creates a new xpath.NodeNavigator for the specified html.Node.
func CreateXPathNavigator(top *html.Node) xpath.NodeNavigator {
	return &htmlNodeNavigator{curr: top, root: top, attr: -1}
}

// Find searches the html.Node that matches by the specified XPath expr.
func Find(top *html.Node, expr string) []*html.Node {
	var elems []*html.Node
	t := gxpath.Select(CreateXPathNavigator(top), expr)
	for t.MoveNext() {
		elems = append(elems, (t.Current().(*htmlNodeNavigator)).curr)
	}
	return elems
}

// FindOne searches the html.Node that matches by the specified XPath expr,
// and returns first element of matched html.Node.
func FindOne(top *html.Node, expr string) *html.Node {
	var elem *html.Node
	t := gxpath.Select(CreateXPathNavigator(top), expr)
	if t.MoveNext() {
		elem = (t.Current().(*htmlNodeNavigator)).curr
	}
	return elem
}

// FindEach searches the html.Node and calls functions cb.
func FindEach(top *html.Node, expr string, cb func(int, *html.Node)) {
	t := gxpath.Select(CreateXPathNavigator(top), expr)
	i := 0
	for t.MoveNext() {
		cb(i, (t.Current().(*htmlNodeNavigator)).curr)
		i++
	}
}

func Load(url string) (*html.Node, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	charset := "utf-8"
	contyp := strings.ToLower(resp.Header.Get("Content-Type"))
	if i := strings.Index(contyp, "charset="); i > 0 {
		charset = contyp[i+8:]
	}
	e, err := htmlindex.Get(charset)
	if err != nil {
		return nil, err
	}
	if name, _ := htmlindex.Name(e); name != "utf8" {
		body := e.NewDecoder().Reader(resp.Body)
		return html.Parse(body)
	}
	return html.Parse(resp.Body)
}

// InnerText returns the text between the start and end tags of the object.
func InnerText(n *html.Node) string {
	if n.Type == html.TextNode || n.Type == html.CommentNode {
		return n.Data
	}
	var buf bytes.Buffer
	for child := n.FirstChild; child != nil; child = child.NextSibling {
		buf.WriteString(InnerText(child))
	}
	return strings.TrimSpace(buf.String())
}

func isSelfClosingTag(t string) bool {
	switch t {
{'fraction_non_alphanumeric': 0.14842059714409347, 'fraction_numerical': 0.0025962786672436176, 'mean_word_length': 4.108287292817679, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2585790', 'n_tokens_mistral': 1739, 'n_tokens_neox': 1680, 'n_words': 433}