<commit_before>package xml

/*
#cgo pkg-config: libxml-2.0

#include "helper.h"
*/
import "C"

import (
	"errors"
	"github.com/moovweb/gokogiri/help"
	. "github.com/moovweb/gokogiri/util"
	"github.com/moovweb/gokogiri/xpath"
	"os"
	"unsafe"
)

type Document interface {
	/* Nokogiri APIs */
	CreateElementNode(string) *ElementNode
	CreateCDataNode(string) *CDataNode
	CreateTextNode(string) *TextNode
	CreateCommentNode(string) *CommentNode
	CreatePINode(string, string) *ProcessingInstructionNode
	ParseFragment([]byte, []byte, ParseOption) (*DocumentFragment, error)

	DocPtr() unsafe.Pointer
	DocType() NodeType
	DocRef() Document
	InputEncoding() []byte
	OutputEncoding() []byte
	DocXPathCtx() *xpath.XPath
	AddUnlinkedNode(unsafe.Pointer)
	RemoveUnlinkedNode(unsafe.Pointer) bool
	Free()
	String() string
	Root() *ElementNode
	NodeById(string) *ElementNode
	BookkeepFragment(*DocumentFragment)

	RecursivelyRemoveNamespaces() error
	UnparsedEntityURI(string) string
}

// ParseOption values allow you to tune the behaviour of the parsing engine.
type ParseOption int

const (
	XML_PARSE_RECOVER    ParseOption = 1 << iota // recover on errors
	XML_PARSE_NOENT                              // substitute entities
	XML_PARSE_DTDLOAD                            // load the external subset
	XML_PARSE_DTDATTR                            // default DTD attributes
	XML_PARSE_DTDVALID                           // validate with the DTD
	XML_PARSE_NOERROR                            // suppress error reports
	XML_PARSE_NOWARNING                          // suppress warning reports
	XML_PARSE_PEDANTIC                           // pedantic error reporting
	XML_PARSE_NOBLANKS                           // remove blank nodes
	XML_PARSE_SAX1                               // use the SAX1 interface internally
	XML_PARSE_XINCLUDE                           // Implement XInclude substitition
	XML_PARSE_NONET                              // Forbid network access
	XML_PARSE_NODICT                             // Do not reuse the context dictionnary
	XML_PARSE_NSCLEAN                            // remove redundant namespaces declarations
	XML_PARSE_NOCDATA                            // merge CDATA as text nodes
	XML_PARSE_NOXINCNODE                         // do not generate XINCLUDE START/END nodes
	XML_PARSE_COMPACT                            // compact small text nodes; makes tree read-only
	XML_PARSE_OLD10                              // parse using XML-1.0 before update 5
	XML_PARSE_NOBASEFIX                          // do not fixup XINCLUDE xml//base uris
	XML_PARSE_HUGE                               // relax any hardcoded limit from the parser
	XML_PARSE_OLDSAX                             // parse using SAX2 interface before 2.7.0
	XML_PARSE_IGNORE_ENC                         // ignore internal document encoding hint
	XML_PARSE_BIG_LINES                          // Store big lines numbers in text PSVI field
)

//DefaultParseOption provides liberal parsing highly tolerant of invalid documents. Errors and warnings
// are suppressed and the DTD is not processed.
const DefaultParseOption ParseOption = XML_PARSE_RECOVER |
	XML_PARSE_NONET |
	XML_PARSE_NOERROR |
	XML_PARSE_NOWARNING

//StrictParseOption provides standard-compliant parsing. The DTD is processed, entity
// substitions are made, and errors and warnings are reported back.
const StrictParseOption ParseOption = XML_PARSE_NOENT |
	XML_PARSE_DTDLOAD |
	XML_PARSE_DTDATTR |
	XML_PARSE_NOCDATA

//DefaultEncoding is UTF-8, which is also the default for both libxml2 and Go strings.
const DefaultEncoding = "utf-8"

var ERR_FAILED_TO_PARSE_XML = errors.New("failed to parse xml input")

/*
XmlDocument is the primary interface for working with XML documents.
*/
type XmlDocument struct {
	Ptr *C.xmlDoc
	Me  Document
	Node
	InEncoding    []byte
	OutEncoding   []byte
	UnlinkedNodes map[*C.xmlNode]bool
	XPathCtx      *xpath.XPath
	Type          NodeType
	InputLen      int

	fragments []*DocumentFragment //save the pointers to free them when the doc is freed
}

//DefaultEncodingBytes allows us to conveniently pass the DefaultEncoding to various functions that
// expect the encoding as a byte array.
const DefaultEncodingBytes = []byte(DefaultEncoding)

const initialFragments = 2

//NewDocument wraps the pointer to the C struct.

// TODO: this should probably not be exported.
func NewDocument(p unsafe.Pointer, contentLen int, inEncoding, outEncoding []byte) (doc *XmlDocument) {
	inEncoding = AppendCStringTerminator(inEncoding)
	outEncoding = AppendCStringTerminator(outEncoding)

	xmlNode := &XmlNode{Ptr: (*C.xmlNode)(p)}
	docPtr := (*C.xmlDoc)(p)
	doc = &XmlDocument{Ptr: docPtr, Node: xmlNode, InEncoding: inEncoding, OutEncoding: outEncoding, InputLen: contentLen}<commit_msg>can't have a const byte array in go1.2, needs to be a var -- issue #66<commit_after>package xml

/*
#cgo pkg-config: libxml-2.0

#include "helper.h"
*/
import "C"

import (
	"errors"
	"github.com/moovweb/gokogiri/help"
	. "github.com/moovweb/gokogiri/util"
	"github.com/moovweb/gokogiri/xpath"
	"os"
	"unsafe"
)

type Document interface {
	/* Nokogiri APIs */
	CreateElementNode(string) *ElementNode
	CreateCDataNode(string) *CDataNode
	CreateTextNode(string) *TextNode
	CreateCommentNode(string) *CommentNode
	CreatePINode(string, string) *ProcessingInstructionNode
	ParseFragment([]byte, []byte, ParseOption) (*DocumentFragment, error)

	DocPtr() unsafe.Pointer
	DocType() NodeType
	DocRef() Document
	InputEncoding() []byte
	OutputEncoding() []byte
	DocXPathCtx() *xpath.XPath
	AddUnlinkedNode(unsafe.Pointer)
	RemoveUnlinkedNode(unsafe.Pointer) bool
	Free()
	String() string
	Root() *ElementNode
	NodeById(string) *ElementNode
	BookkeepFragment(*DocumentFragment)

	RecursivelyRemoveNamespaces() error
	UnparsedEntityURI(string) string
}

// ParseOption values allow you to tune the behaviour of the parsing engine.
type ParseOption int

const (
	XML_PARSE_RECOVER    ParseOption = 1 << iota // recover on errors
	XML_PARSE_NOENT                              // substitute entities
	XML_PARSE_DTDLOAD                            // load the external subset
	XML_PARSE_DTDATTR                            // default DTD attributes
	XML_PARSE_DTDVALID                           // validate with the DTD
	XML_PARSE_NOERROR                            // suppress error reports
	XML_PARSE_NOWARNING                          // suppress warning reports
	XML_PARSE_PEDANTIC                           // pedantic error reporting
	XML_PARSE_NOBLANKS                           // remove blank nodes
	XML_PARSE_SAX1                               // use the SAX1 interface internally
	XML_PARSE_XINCLUDE                           // Implement XInclude substitition
	XML_PARSE_NONET                              // Forbid network access
	XML_PARSE_NODICT                             // Do not reuse the context dictionnary
	XML_PARSE_NSCLEAN                            // remove redundant namespaces declarations
	XML_PARSE_NOCDATA                            // merge CDATA as text nodes
	XML_PARSE_NOXINCNODE                         // do not generate XINCLUDE START/END nodes
	XML_PARSE_COMPACT                            // compact small text nodes; makes tree read-only
	XML_PARSE_OLD10                              // parse using XML-1.0 before update 5
	XML_PARSE_NOBASEFIX                          // do not fixup XINCLUDE xml//base uris
	XML_PARSE_HUGE                               // relax any hardcoded limit from the parser
	XML_PARSE_OLDSAX                             // parse using SAX2 interface before 2.7.0
	XML_PARSE_IGNORE_ENC                         // ignore internal document encoding hint
	XML_PARSE_BIG_LINES                          // Store big lines numbers in text PSVI field
)

//DefaultParseOption provides liberal parsing highly tolerant of invalid documents. Errors and warnings
// are suppressed and the DTD is not processed.
const DefaultParseOption ParseOption = XML_PARSE_RECOVER |
	XML_PARSE_NONET |
	XML_PARSE_NOERROR |
	XML_PARSE_NOWARNING

//StrictParseOption provides standard-compliant parsing. The DTD is processed, entity
// substitions are made, and errors and warnings are reported back.
const StrictParseOption ParseOption = XML_PARSE_NOENT |
	XML_PARSE_DTDLOAD |
	XML_PARSE_DTDATTR |
	XML_PARSE_NOCDATA

//DefaultEncoding is UTF-8, which is also the default for both libxml2 and Go strings.
const DefaultEncoding = "utf-8"

var ERR_FAILED_TO_PARSE_XML = errors.New("failed to parse xml input")

/*
XmlDocument is the primary interface for working with XML documents.
*/
type XmlDocument struct {
	Ptr *C.xmlDoc
	Me  Document
	Node
	InEncoding    []byte
	OutEncoding   []byte
	UnlinkedNodes map[*C.xmlNode]bool
	XPathCtx      *xpath.XPath
	Type          NodeType
	InputLen      int

	fragments []*DocumentFragment //save the pointers to free them when the doc is freed
}

//DefaultEncodingBytes allows us to conveniently pass the DefaultEncoding to various functions that
// expect the encoding as a byte array.
var DefaultEncodingBytes = []byte(DefaultEncoding)

const initialFragments = 2

//NewDocument wraps the pointer to the C struct.

// TODO: this should probably not be exported.
func NewDocument(p unsafe.Pointer, contentLen int, inEncoding, outEncoding []byte) (doc *XmlDocument) {
	inEncoding = AppendCStringTerminator(inEncoding)
	outEncoding = AppendCStringTerminator(outEncoding)

	xmlNode := &XmlNode{Ptr: (*C.xmlNode)(p)}
	docPtr := (*C.xmlDoc)(p)
	doc = &XmlDocument{Ptr: docPtr, Node: xmlNode, InEncoding: inEncoding, OutEncoding: outEncoding, InputLen: contentLen}
{'fraction_non_alphanumeric': 0.06633983519349118, 'fraction_numerical': 0.004172316678835924, 'mean_word_length': 2.669345579793341, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4577116', 'n_tokens_mistral': 2806, 'n_tokens_neox': 2726, 'n_words': 815}