<commit_before>package main

import (
	"fmt"
)

type NodeType int

const (
	NodeNil NodeType = iota
	NodeRoot
	NodeTag
	NodeText
	NodeComment
	NodeDoctype
)

var NodeTypeString = map[NodeType]string {
	NodeNil:		"Nil",
	NodeRoot:		"Root",
	NodeTag:		"Tag",
	NodeText:		"Text",
	NodeComment:	"Comment",
	NodeDoctype:	"Doctype",
}

type Node struct {
	Type NodeType

	parent *Node

	// were tags printed out? important to know if NodeText follows and needs
	// to append more text
	opened bool
	closed bool

	tag string
	attrs map[string]string
	text string

	// dumb appended string while attribute assignment is still being determined
	// this is used if it turns out tokens are NOT part of an attribute assignment
	// and the string (with whitespace if any) gets used as the node's text
	attrString string
}

func (n *Node) TypeString() string {
	if nodeType, found := NodeTypeString[n.Type]; found {
		return nodeType
	} else {
		return "???"
	}
}

func (n *Node) OpenString() string {
	output := ""
	if n.Type == NodeTag && !n.opened {
		output += "<" + n.tag
		for k, v := range n.attrs {
			output += fmt.Sprintf(" %s='%s'", k, v)
		}
		output += ">"
		n.opened = true
	}
	if n.Type == NodeTag || n.Type == NodeText {
		output += n.attrString
		output += n.text
		n.attrString = ""
	}
	return output
}

func (n *Node) CloseString() string {
	output := ""
	if n.Type == NodeTag && !n.closed {
		output += "</" + n.tag + ">"
		n.closed = true
	}
	return output<commit_msg>outputting node text resets the property to an empty string<commit_after>package main

import (
	"fmt"
)

type NodeType int

const (
	NodeNil NodeType = iota
	NodeRoot
	NodeTag
	NodeText
	NodeComment
	NodeDoctype
)

var NodeTypeString = map[NodeType]string {
	NodeNil:		"Nil",
	NodeRoot:		"Root",
	NodeTag:		"Tag",
	NodeText:		"Text",
	NodeComment:	"Comment",
	NodeDoctype:	"Doctype",
}

type Node struct {
	Type NodeType

	parent *Node

	// were tags printed out? important to know if NodeText follows and needs
	// to append more text
	opened bool
	closed bool

	tag string
	attrs map[string]string
	text string

	// dumb appended string while attribute assignment is still being determined
	// this is used if it turns out tokens are NOT part of an attribute assignment
	// and the string (with whitespace if any) gets used as the node's text
	attrString string
}

func (n *Node) TypeString() string {
	if nodeType, found := NodeTypeString[n.Type]; found {
		return nodeType
	} else {
		return "???"
	}
}

func (n *Node) OpenString() string {
	output := ""
	if n.Type == NodeTag && !n.opened {
		output += "<" + n.tag
		for k, v := range n.attrs {
			output += fmt.Sprintf(" %s='%s'", k, v)
		}
		output += ">"
		n.opened = true
	}
	if n.Type == NodeTag || n.Type == NodeText {
		output += n.attrString
		output += n.text
		n.attrString = ""
		n.text = ""
	}
	return output
}

func (n *Node) CloseString() string {
	output := ""
	if n.Type == NodeTag && !n.closed {
		output += "</" + n.tag + ">"
		n.closed = true
	}
	return output
{'fraction_non_alphanumeric': 0.12624584717607973, 'fraction_numerical': 0.0, 'mean_word_length': 3.3014285714285716, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '541865', 'n_tokens_mistral': 1100, 'n_tokens_neox': 1048, 'n_words': 351}