<commit_before>package crawler

import (
	"errors"
	"strings"
	"unicode"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

var (
	// ErrMinificationUnexpectedNodeType - found unexpected node type
	ErrMinificationUnexpectedNodeType = errors.New("minification.Minification.parseNode: unexpected node type")
)

// Minification - struct with functions for minimize html
type Minification struct {
}

func (m *Minification) removeAttr(node *html.Node) {
	lenAttr := len(node.Attr)
	if lenAttr != 0 {
		attr := node.Attr
		i := 0
		j := 0
		for ; i != lenAttr; i++ {
			key := strings.ToLower(attr[i].Key)
			if strings.HasPrefix(key, "data-") {
				continue
			}
			switch key {
			case "id":
			case "alt":
			case "cols":
			case "class":
			case "title":
			case "width":
			case "align":
			case "style":
			case "color":
			case "valign":
			case "target":
			case "height":
			case "border":
			case "hspace":
			case "vspace":
			case "bgcolor":
			case "onclick":
			case "colspan":
			case "itemprop":
			case "disabled":
			case "itemtype":
			case "itemscope":
			case "cellspacing":
			case "cellpadding":
			case "bordercolor":
			default:
				if i != j {
					attr[j] = attr[i]
				}
				j++
			}
		}
		if i != j {
			node.Attr = attr[:j]
		}
	}
}

func (m *Minification) removeNode(node *html.Node) (*html.Node, error) {
	parent := node.Parent
	prev := node.PrevSibling
	prevText := prev != nil && prev.Type == html.TextNode
	next := node.NextSibling
	nextText := next != nil && next.Type == html.TextNode
	var result *html.Node

	if prevText && nextText {
		prev.Data = strings.TrimRightFunc(prev.Data, unicode.IsSpace) +
			" " + strings.TrimLeftFunc(next.Data, unicode.IsSpace)
		parent.RemoveChild(node)
		parent.RemoveChild(next)
		result = prev.NextSibling
	} else if prevText {
		prev.Data = strings.TrimRightFunc(prev.Data, unicode.IsSpace) + " "
		parent.RemoveChild(node)
		result = prev.NextSibling
	} else if nextText {
		next.Data = " " + strings.TrimLeftFunc(next.Data, unicode.IsSpace)
		parent.RemoveChild(node)
		result = next
	} else {
		node.Type = html.TextNode
		node.Data = " "
		node.FirstChild = nil
		node.LastChild = nil
		result = next
	}

	return result, nil
}

func (m *Minification) parseChildren(node *html.Node) (*html.Node, error) {
	var err error
	for it := node.FirstChild; it != nil; {
		it, err = m.parseNode(it)
		if err != nil {
			return node.NextSibling, err
		}
	}

	return node.NextSibling, nil
}

func (m *Minification) parseElements(node *html.Node) (*html.Node, error) {
	switch node.DataAtom {
	case atom.Script:
		return m.removeNode(node)
	case atom.Style:
		return m.removeNode(node)
	case atom.Form:
		return m.removeNode(node)
	case atom.Button:
		return m.removeNode(node)
	case atom.Img:
		return m.removeNode(node)
	case atom.Time:
		return m.removeNode(node)
	case atom.Br:
		return m.removeNode(node)
	case atom.Hr:
		return m.removeNode(node)
	}

	m.removeAttr(node)

	return m.parseChildren(node)
}

func (m *Minification) parseNode(node *html.Node) (*html.Node, error) {
	switch node.Type {
	case html.DocumentNode: // +children -attr (first node)
		return m.parseChildren(node)
	case html.ElementNode: // +children +attr
		return m.parseElements(node)
	case html.TextNode: // -children -attr
		return nil, nil
	case html.DoctypeNode: // ignore
		return nil, nil
	case html.CommentNode: // remove
		return m.removeNode(node)
	default:
		return nil, ErrMinificationUnexpectedNodeType
	}
}

// Run - start minification node
func (m *Minification) Run(node *html.Node) error {
	_, err := parseNode(node)
	return err
}<commit_msg>ошибки с обходом дерева в минификаторе<commit_after>package crawler

import (
	"errors"
	"strings"
	"unicode"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

var (
	// ErrMinificationUnexpectedNodeType - found unexpected node type
	ErrMinificationUnexpectedNodeType = errors.New("minification1.minification1.parseNode: unexpected node type")
)

type minification1 struct {
}

func (m *minification1) removeAttr(node *html.Node) {
	lenAttr := len(node.Attr)
	if lenAttr != 0 {
		attr := node.Attr
		i := 0
		j := 0
		for ; i != lenAttr; i++ {
			key := strings.ToLower(attr[i].Key)
			if strings.HasPrefix(key, "data-") {
				continue
			}
			switch key {
			case "id":
			case "alt":
			case "cols":
			case "class":
			case "title":
			case "width":
			case "align":
			case "style":
			case "color":
			case "valign":
			case "target":
			case "height":
			case "border":
			case "hspace":
			case "vspace":
			case "bgcolor":
			case "onclick":
			case "colspan":
			case "itemprop":
			case "disabled":
			case "itemtype":
			case "itemscope":
			case "cellspacing":
			case "cellpadding":
			case "bordercolor":
			default:
				if i != j {
					attr[j] = attr[i]
				}
				j++
			}
		}
		if i != j {
			node.Attr = attr[:j]
		}
	}
}

func (m *minification1) removeNode(node *html.Node) (*html.Node, error) {
	parent := node.Parent
	prev := node.PrevSibling
	prevText := prev != nil && prev.Type == html.TextNode
	next := node.NextSibling
	nextText := next != nil && next.Type == html.TextNode
	var result *html.Node

	if prevText && nextText {
		prev.Data = strings.TrimRightFunc(prev.Data, unicode.IsSpace) +
			" " + strings.TrimLeftFunc(next.Data, unicode.IsSpace)
		parent.RemoveChild(node)
		parent.RemoveChild(next)
		result = prev.NextSibling
	} else if prevText {
		prev.Data = strings.TrimRightFunc(prev.Data, unicode.IsSpace) + " "
		parent.RemoveChild(node)
		result = prev.NextSibling
	} else if nextText {
		next.Data = " " + strings.TrimLeftFunc(next.Data, unicode.IsSpace)
		parent.RemoveChild(node)
		result = next
	} else {
		node.Type = html.TextNode
		node.Data = " "
		node.FirstChild = nil
		node.LastChild = nil
		result = next
	}

	return result, nil
}

func (m *minification1) parseChildren(node *html.Node) (*html.Node, error) {
	var err error
	for it := node.FirstChild; it != nil; {
		it, err = m.parseNode(it)
		if err != nil {
			return node.NextSibling, err
		}
	}

	return node.NextSibling, nil
}

func (m *minification1) parseElements(node *html.Node) (*html.Node, error) {
	switch node.DataAtom {
	case atom.Script:
		return m.removeNode(node)
	case atom.Style:
		return m.removeNode(node)
	case atom.Form:
		return m.removeNode(node)
	case atom.Button:
		return m.removeNode(node)
	case atom.Img:
		return m.removeNode(node)
	case atom.Time:
		return m.removeNode(node)
	case atom.Br:
		return m.removeNode(node)
	case atom.Hr:
		return m.removeNode(node)
	}

	m.removeAttr(node)

	return m.parseChildren(node)
}

func (m *minification1) parseNode(node *html.Node) (*html.Node, error) {
	switch node.Type {
	case html.DocumentNode: // +children -attr (first node)
		return m.parseChildren(node)
	case html.ElementNode: // +children +attr
		return m.parseElements(node)
	case html.TextNode: // -children -attr
		return node.NextSibling, nil
	case html.DoctypeNode: // ignore
		return node.NextSibling, nil
	case html.CommentNode: // remove
		return m.removeNode(node)
	default:
		return nil, ErrMinificationUnexpectedNodeType
	}
}

// RunMinification1 - start minification node
func RunMinification1(node *html.Node) error {
	m := minification1{}
	_, err := m.parseNode(node)
	return err
}
{'fraction_non_alphanumeric': 0.1354411155598509, 'fraction_numerical': 0.0023470937456854892, 'mean_word_length': 3.4799010513296227, 'pattern_counts': {'":': 50, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2098091', 'n_tokens_mistral': 2861, 'n_tokens_neox': 2586, 'n_words': 634}