<commit_before>package terse

import (
	"fmt"
	"regexp"
	"strings"
)

func parseTag(node *rawNode, children bool) (*tag, error) {
	t := &tag{
		Name:     "div",
		Node:     node,
		Source:   node.Code,
		Attrs:    make(map[string]string),
		DynAttrs: make(map[string]string),
	}
	e := t.Parse(children)
	return t, e
}

type tag struct {
	Node      *rawNode
	Source    string
	Name      string
	ChildTags []string
	Id        string
	Classes   []string
	Attrs     map[string]string
	DynAttrs  map[string]string
	Remaining string
	Enclosing bool
}

var attrStartRegex = regexp.MustCompile(`^([a-zA-Z0-9_-]+)=`)

func (t *tag) Parse(children bool) error {
	t.Enclosing = children
	// pull percentage sign off
	if t.Source[0] == '%' {
		t.Source = t.Source[1:]
	}
	// pull element name
	if t.Source[0] != '#' && t.Source[0] != '.' {
		t.Name = firstTextToken(t.Source)
		t.Source = t.Source[len(t.Name):]
	}
	// pull id and class static's
	for len(t.Source) > 0 && (t.Source[0] == '#' || t.Source[0] == '.') {
		switch t.Source[0] {
		case '#':
			t.Id = firstTextToken(t.Source[1:])
			t.Source = t.Source[len(t.Id)+1:]
		case '.':
			cl := firstTextToken(t.Source[1:])
			t.Source = t.Source[len(cl)+1:]
			t.Classes = append(t.Classes, cl)
		}
	}
	if !t.Enclosing {
		if !VoidElements[t.Name] {
			t.Enclosing = true
		}
	}

	if len(t.Source) > 0 && t.Source[0] == '(' {
		if len(t.Node.Children) == 0 {
			return fmt.Errorf("Missing nested attributes for %s", t.Node.Code)
		}
		for _, child := range t.Node.Children {
			if attrStartRegex.MatchString(strings.TrimSpace(child.Code)) {
				t.Source = child.Code
				e := t.parseAttributes()
				if e != nil {
					return e
				}
			} else if strings.HasPrefix(strings.TrimSpace(child.Code), ")") {
				t.Node.Children = t.Node.Children[1:]
				t.Remaining = strings.TrimSpace(child.Code)[1:]
				break
			}

			t.Node.Children = t.Node.Children[1:]
			if strings.HasPrefix(strings.TrimSpace(t.Source), ")") {
				t.Remaining = strings.TrimSpace(t.Source)[1:]
				break
			}
			if t.Remaining != "" {
				break
			}
		}
	} else if attrStartRegex.MatchString(strings.TrimSpace(t.Source)) {
		e := t.parseAttributes()
		if e != nil {
			return e
		}
	}
	t.Remaining = t.Source
	if len(t.Remaining) > 0 && t.Remaining[0] == ' ' {
		t.Remaining = t.Remaining[1:]
	}
	for len(t.Remaining) > 0 && t.Remaining[0] == '>' {
		check := strings.TrimSpace(t.Remaining[1:])
		switch {
		case len(check) == 0:
			return fmt.Errorf("Missing tag in nested tags for %s", t.Node.Code)
		case check[0] == '%':
			element := firstTextToken(check[1:])
			check = check[len(element)+1:]
			t.Remaining = strings.TrimSpace(check)
			t.ChildTags = append(t.ChildTags, element)
		case ValidElements[firstTextToken(check)]:
			element := firstTextToken(check)
			check = check[len(element):]
			t.Remaining = strings.TrimSpace(check)
			t.ChildTags = append(t.ChildTags, element)
		}
	}
	return nil
}<commit_msg>Fix for totem tags, classes and ids now work for non-start tags<commit_after>package terse

import (
	"fmt"
	"regexp"
	"strings"
)

func parseTag(node *rawNode, children bool) (*tag, error) {
	t := &tag{
		Name:     "div",
		Node:     node,
		Source:   node.Code,
		Attrs:    make(map[string]string),
		DynAttrs: make(map[string]string),
	}
	e := t.Parse(children)
	return t, e
}

type tag struct {
	Node      *rawNode
	Source    string
	Name      string
	ChildTags []string
	Id        string
	Classes   []string
	Attrs     map[string]string
	DynAttrs  map[string]string
	Remaining string
	Enclosing bool
}

var attrStartRegex = regexp.MustCompile(`^([a-zA-Z0-9_-]+)=`)

func (t *tag) Parse(children bool) error {
	t.Enclosing = children
	// pull percentage sign off
	if t.Source[0] == '%' {
		t.Source = t.Source[1:]
	}
	// pull element name
	if t.Source[0] != '#' && t.Source[0] != '.' {
		t.Name = firstTextToken(t.Source)
		t.Source = t.Source[len(t.Name):]
	}
	// pull id and class static's
	for len(t.Source) > 0 && (t.Source[0] == '#' || t.Source[0] == '.') {
		switch t.Source[0] {
		case '#':
			t.Id = firstTextToken(t.Source[1:])
			t.Source = t.Source[len(t.Id)+1:]
		case '.':
			cl := firstTextToken(t.Source[1:])
			t.Source = t.Source[len(cl)+1:]
			t.Classes = append(t.Classes, cl)
		}
	}
	if !t.Enclosing {
		if !VoidElements[t.Name] {
			t.Enclosing = true
		}
	}

	if len(t.Source) > 0 && t.Source[0] == '(' {
		if len(t.Node.Children) == 0 {
			return fmt.Errorf("Missing nested attributes for %s", t.Node.Code)
		}
		for _, child := range t.Node.Children {
			if attrStartRegex.MatchString(strings.TrimSpace(child.Code)) {
				t.Source = child.Code
				e := t.parseAttributes()
				if e != nil {
					return e
				}
			} else if strings.HasPrefix(strings.TrimSpace(child.Code), ")") {
				t.Node.Children = t.Node.Children[1:]
				t.Remaining = strings.TrimSpace(child.Code)[1:]
				break
			}

			t.Node.Children = t.Node.Children[1:]
			if strings.HasPrefix(strings.TrimSpace(t.Source), ")") {
				t.Remaining = strings.TrimSpace(t.Source)[1:]
				break
			}
			if t.Remaining != "" {
				break
			}
		}
	} else if attrStartRegex.MatchString(strings.TrimSpace(t.Source)) {
		e := t.parseAttributes()
		if e != nil {
			return e
		}
	}
	t.Remaining = t.Source
	if len(t.Remaining) > 0 && t.Remaining[0] == ' ' {
		t.Remaining = t.Remaining[1:]
	}
	// totem tags
	if len(t.Remaining) > 0 && t.Remaining[0] == '>' {
		check := strings.TrimSpace(t.Remaining[1:])
		switch {
		case len(check) == 0:
			return fmt.Errorf("Missing tag in nested tags for %s", t.Node.Code)
		case check[0] == '%':
			t.Remaining = strings.TrimSpace(check)
		case ValidElements[firstTextToken(check)]:
			t.Remaining = strings.TrimSpace(check)
		}
	}
	return nil
}
{'fraction_non_alphanumeric': 0.1738061797752809, 'fraction_numerical': 0.010533707865168539, 'mean_word_length': 3.089734386216798, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7417586', 'n_tokens_mistral': 2465, 'n_tokens_neox': 2190, 'n_words': 441}