<commit_before>package ace

import (
	"bytes"
	"fmt"
	"io"
	"strings"
)

// Tag names
const (
	tagNameBr   = "br"
	tagNameDiv  = "div"
	tagNameMeta = "meta"
)

// Attribute names
const (
	attributeNameID    = "id"
	attributeNameClass = "class"
)

// htmlTag represents an HTML tag.
type htmlTag struct {
	elementBase
	tagName          string
	id               string
	classes          []string
	containPlainText bool
	insertBr         bool
	attributes       map[string]string
	textValue        string
}

// WriteTo writes data to w.
func (e *htmlTag) WriteTo(w io.Writer) (int64, error) {
	var bf bytes.Buffer

	// Write an open tag.
	bf.WriteString(lt)
	bf.WriteString(e.tagName)
	// Write an id.
	if e.id != "" {
		bf.WriteString(space)
		bf.WriteString(attributeNameID)
		bf.WriteString(equal)
		bf.WriteString(doubleQuote)
		bf.WriteString(e.id)
		bf.WriteString(doubleQuote)
	}
	// Write classes.
	if len(e.classes) > 0 {
		bf.WriteString(space)
		bf.WriteString(attributeNameClass)
		bf.WriteString(equal)
		bf.WriteString(doubleQuote)
		for i, class := range e.classes {
			if i > 0 {
				bf.WriteString(space)
			}
			bf.WriteString(class)
		}
		bf.WriteString(doubleQuote)
	}
	// Write attributes.
	if len(e.attributes) > 0 {
		for k, v := range e.attributes {
			bf.WriteString(space)
			bf.WriteString(k)
			if v != "" {
				bf.WriteString(equal)
				bf.WriteString(doubleQuote)
				bf.WriteString(v)
				bf.WriteString(doubleQuote)
			}
		}
	}
	bf.WriteString(gt)

	// Write a text value
	if e.textValue != "" {
		bf.WriteString(e.textValue)
	}

	if e.containPlainText {
		bf.WriteString(lf)
	}

	// Write children's HTML.
	if i, err := e.writeChildren(&bf); err != nil {
		return i, err
	}

	// Write a close tag.
	if e.tagName != tagNameBr && e.tagName != tagNameMeta {
		bf.WriteString(lt)
		bf.WriteString(slash)
		bf.WriteString(e.tagName)
		bf.WriteString(gt)
	}

	// Write the buffer.
	i, err := w.Write(bf.Bytes())

	return int64(i), err
}

// ContainPlainText returns the HTML tag's containPlainText field.
func (e *htmlTag) ContainPlainText() bool {
	return e.containPlainText
}

// InsertBr returns true if the br tag is inserted to the line.
func (e *htmlTag) InsertBr() bool {
	return e.insertBr
}

// setAttributes parses the tokens and set attributes to the element.
func (e *htmlTag) setAttributes() error {
	var parsedTokens []string
	var unclosedTokenValues []string
	var unclosed bool
	var closeMark string

	for i, token := range e.ln.tokens {
		if i == 0 {
			continue
		}
		if unclosed {
			unclosedTokenValues = append(unclosedTokenValues, token)

			if closed(token, closeMark) {
				parsedTokens = append(parsedTokens, strings.Join(unclosedTokenValues, space))
				unclosedTokenValues = make([]string, 0)
				unclosed = false
			}
		} else {
			if unclosed, closeMark = unclosedToken(token, e.opts); unclosed {
				unclosedTokenValues = append(unclosedTokenValues, token)
			} else {
				parsedTokens = append(parsedTokens, token)
			}
		}
	}

	if unclosed {
		parsedTokens = append(parsedTokens, unclosedTokenValues...)
	}

	var i int
	var token string
	var setTextValue bool

	// Set attributes to the element.
	for i, token = range parsedTokens {
		kv := strings.Split(token, equal)

		if len(kv) < 2 {
			setTextValue = true
			break
		}

		k := kv[0]
		v := kv[1]

		// Remove the prefix and suffix of the double quotes.
		if len(v) > 1 && strings.HasPrefix(v, doubleQuote) && strings.HasSuffix(v, doubleQuote) {
			v = v[1 : len(v)-1]
		}

		switch k {
		case attributeNameID:
			if e.id != "" {
				return fmt.Errorf("multiple IDs are specified [file: %s][line: %d]", e.ln.fileName(), e.ln.no)
			}
			e.id = v
		case attributeNameClass:
			e.classes = append(e.classes, strings.Split(v, space)...)
		default:
			e.attributes[k] = v
		}
	}

	// Set a text value to the element.
	if setTextValue {
		e.textValue = strings.Join(parsedTokens[i:], space)
	}

	return nil
}

// newHTMLTag creates and returns an HTML tag.
func newHTMLTag(ln *line, rslt *result, src *source, parent element, opts *Options) (*htmlTag, error) {
	if len(ln.tokens) < 1 {
		return nil, fmt.Errorf("an HTML tag is not specified [file: %s][line: %d]", ln.fileName(), ln.no)
	}

	s := ln.tokens[0]

	tagName := extractTagName(s)

	id, err := extractID(s, ln)
	if err != nil {
		return nil, err
	}

	classes := extractClasses(s)

	e := &htmlTag{
		elementBase:      newElementBase(ln, rslt, src, parent, opts),
		tagName:          tagName,
		id:               id,
		classes:          classes,
		containPlainText: strings.HasSuffix(s, dot),<commit_msg>Update html_tag.go<commit_after>package ace

import (
	"bytes"
	"fmt"
	"io"
	"strings"
)

// Tag names
const (
	tagNameBr    = "br"
	tagNameDiv   = "div"
	tagNameHr    = "hr"
	tagNameImg   = "img"
	tagNameInput = "input"
	tagNameLink  = "link"
	tagNameMeta  = "meta"
)

var noCloseTagNames = []string{
	tagNameBr,
	tagNameHr,
	tagNameImg,
	tagNameInput,
	tagNameLink,
	tagNameMeta,
}

// Attribute names
const (
	attributeNameID    = "id"
	attributeNameClass = "class"
)

// htmlTag represents an HTML tag.
type htmlTag struct {
	elementBase
	tagName          string
	id               string
	classes          []string
	containPlainText bool
	insertBr         bool
	attributes       map[string]string
	textValue        string
}

// WriteTo writes data to w.
func (e *htmlTag) WriteTo(w io.Writer) (int64, error) {
	var bf bytes.Buffer

	// Write an open tag.
	bf.WriteString(lt)
	bf.WriteString(e.tagName)
	// Write an id.
	if e.id != "" {
		bf.WriteString(space)
		bf.WriteString(attributeNameID)
		bf.WriteString(equal)
		bf.WriteString(doubleQuote)
		bf.WriteString(e.id)
		bf.WriteString(doubleQuote)
	}
	// Write classes.
	if len(e.classes) > 0 {
		bf.WriteString(space)
		bf.WriteString(attributeNameClass)
		bf.WriteString(equal)
		bf.WriteString(doubleQuote)
		for i, class := range e.classes {
			if i > 0 {
				bf.WriteString(space)
			}
			bf.WriteString(class)
		}
		bf.WriteString(doubleQuote)
	}
	// Write attributes.
	if len(e.attributes) > 0 {
		for k, v := range e.attributes {
			bf.WriteString(space)
			bf.WriteString(k)
			if v != "" {
				bf.WriteString(equal)
				bf.WriteString(doubleQuote)
				bf.WriteString(v)
				bf.WriteString(doubleQuote)
			}
		}
	}
	bf.WriteString(gt)

	// Write a text value
	if e.textValue != "" {
		bf.WriteString(e.textValue)
	}

	if e.containPlainText {
		bf.WriteString(lf)
	}

	// Write children's HTML.
	if i, err := e.writeChildren(&bf); err != nil {
		return i, err
	}

	// Write a close tag.
	if !e.noCloseTag() {
		bf.WriteString(lt)
		bf.WriteString(slash)
		bf.WriteString(e.tagName)
		bf.WriteString(gt)
	}

	// Write the buffer.
	i, err := w.Write(bf.Bytes())

	return int64(i), err
}

// ContainPlainText returns the HTML tag's containPlainText field.
func (e *htmlTag) ContainPlainText() bool {
	return e.containPlainText
}

// InsertBr returns true if the br tag is inserted to the line.
func (e *htmlTag) InsertBr() bool {
	return e.insertBr
}

// setAttributes parses the tokens and set attributes to the element.
func (e *htmlTag) setAttributes() error {
	var parsedTokens []string
	var unclosedTokenValues []string
	var unclosed bool
	var closeMark string

	for i, token := range e.ln.tokens {
		if i == 0 {
			continue
		}
		if unclosed {
			unclosedTokenValues = append(unclosedTokenValues, token)

			if closed(token, closeMark) {
				parsedTokens = append(parsedTokens, strings.Join(unclosedTokenValues, space))
				unclosedTokenValues = make([]string, 0)
				unclosed = false
			}
		} else {
			if unclosed, closeMark = unclosedToken(token, e.opts); unclosed {
				unclosedTokenValues = append(unclosedTokenValues, token)
			} else {
				parsedTokens = append(parsedTokens, token)
			}
		}
	}

	if unclosed {
		parsedTokens = append(parsedTokens, unclosedTokenValues...)
	}

	var i int
	var token string
	var setTextValue bool

	// Set attributes to the element.
	for i, token = range parsedTokens {
		kv := strings.Split(token, equal)

		if len(kv) < 2 {
			setTextValue = true
			break
		}

		k := kv[0]
		v := kv[1]

		// Remove the prefix and suffix of the double quotes.
		if len(v) > 1 && strings.HasPrefix(v, doubleQuote) && strings.HasSuffix(v, doubleQuote) {
			v = v[1 : len(v)-1]
		}

		switch k {
		case attributeNameID:
			if e.id != "" {
				return fmt.Errorf("multiple IDs are specified [file: %s][line: %d]", e.ln.fileName(), e.ln.no)
			}
			e.id = v
		case attributeNameClass:
			e.classes = append(e.classes, strings.Split(v, space)...)
		default:
			e.attributes[k] = v
		}
	}

	// Set a text value to the element.
	if setTextValue {
		e.textValue = strings.Join(parsedTokens[i:], space)
	}

	return nil
}

// noCloseTag returns true is the HTML tag has no close tag.
func (e *htmlTag) noCloseTag() bool {
	for _, name := range noCloseTagNames {
		if e.tagName == name {
			return true
		}
	}

	return false
}

// newHTMLTag creates and returns an HTML tag.
func newHTMLTag(ln *line, rslt *result, src *source, parent element, opts *Options) (*htmlTag, error) {
	if len(ln.tokens) < 1 {
		return nil, fmt.Errorf("an HTML tag is not specified [file: %s][line: %d]", ln.fileName(), ln.no)
	}

	s := ln.tokens[0]

	tagName := extractTagName(s)

	id, err := extractID(s, ln)
	if err != nil {
		return nil, err
	}

	classes := extractClasses(s)

	e := &htmlTag{
		elementBase:      newElementBase(ln, rslt, src, parent, opts),
		tagName:          tagName,
		id:               id,
		classes:          classes,
		containPlainText: strings.HasSuffix(s, dot),
{'fraction_non_alphanumeric': 0.12468460891505467, 'fraction_numerical': 0.0035744322960470985, 'mean_word_length': 3.089853826311264, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2297030', 'n_tokens_mistral': 3735, 'n_tokens_neox': 3415, 'n_words': 895}