<commit_before>package gxml

import (
	"bytes"
	"encoding/xml"
	"strings"
	"unicode"

	"github.com/rveen/ogdl"
	"golang.org/x/text/runes"
	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
)

func Simplify(g *ogdl.Graph) {
	if g.Out == nil {
		return
	}

	for _, n := range g.Out {

		if len(n.Out) == 0 {
			continue
		}
		if n.Out[0].ThisString() == "@id" && len(n.Out) >= 2 {
			if n.This != "p" {
				n.This = n.Out[0].Out[0].This
			}
			n.Out[0] = n.Out[len(n.Out)-1]
			n.Out = n.Out[:1]
		}
		Simplify(n)
	}
}

func FromXML(b []byte) *ogdl.Graph {

	decoder := xml.NewDecoder(bytes.NewReader(b))

	g := ogdl.New(nil)
	var key string
	level := -1

	att := true

	var stack []*ogdl.Graph
	stack = append(stack, g)

	tr := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)<commit_msg>Simplify field @name NAME to NAME, spaces to underscores<commit_after>package gxml

import (
	"bytes"
	"encoding/xml"
	"strings"
	"unicode"

	"github.com/rveen/ogdl"
	"golang.org/x/text/runes"
	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
)

func Simplify(g *ogdl.Graph) {
	if g.Out == nil {
		return
	}

	for _, n := range g.Out {

		if len(n.Out) == 0 {
			continue
		}
		if n.Out[0].ThisString() == "@id" && len(n.Out) >= 2 {
			if n.This != "p" {
				n.This = n.Out[0].Out[0].This
			}
			n.Out[0] = n.Out[len(n.Out)-1]
			n.Out = n.Out[:1]
		} else if n.Len() > 0 && n.Out[0].Len() == 1 && n.ThisString() == "field" && n.String() == "@name" {
			n.This = strings.ReplaceAll(n.Out[0].Out[0].ThisString(), " ", "_")
			n.Out = n.Out[1:]
		}
		Simplify(n)
	}
}

func FromXML(b []byte) *ogdl.Graph {

	decoder := xml.NewDecoder(bytes.NewReader(b))

	g := ogdl.New(nil)
	var key string
	level := -1

	att := true

	var stack []*ogdl.Graph
	stack = append(stack, g)

	tr := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
{'fraction_non_alphanumeric': 0.20921610169491525, 'fraction_numerical': 0.012711864406779662, 'mean_word_length': 3.0277185501066097, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3165139', 'n_tokens_mistral': 929, 'n_tokens_neox': 828, 'n_words': 153}