<commit_before>// This is free and unencumbered software released into the public
// domain.  For more information, see <http://unlicense.org> or the
// accompanying UNLICENSE file.

package syntax

import (
	"go/ast"
	"go/parser"
	"go/token"

	"github.com/nelsam/gxui"
)

func zeroBasedPos(pos token.Pos) uint64 {
	if pos < 1 {
		panic("Positions of 0 are not valid positions")
	}
	return uint64(pos) - 1
}

func nodeLayer(node ast.Node, colors ...gxui.Color) *gxui.CodeSyntaxLayer {
	return layer(node.Pos(), int(node.End()-node.Pos()), colors...)
}

func layer(pos token.Pos, length int, colors ...gxui.Color) *gxui.CodeSyntaxLayer {
	if length == 0 {
		return nil
	}
	if len(colors) == 0 {
		panic("No colors passed to layer()")
	}
	if len(colors) > 2 {
		panic("Only two colors (text and background) are currently supported")
	}
	layer := gxui.CreateCodeSyntaxLayer()
	layer.Add(int(zeroBasedPos(pos)), length)
	layer.SetColor(colors[0])
	if len(colors) > 1 {
		layer.SetBackgroundColor(colors[1])
	}
	return layer
}

// TODO: highlight matching parenthesis/braces/brackets
func Layers(filename, text string) (gxui.CodeSyntaxLayers, error) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, filename, text, 0)
	layers := make(gxui.CodeSyntaxLayers, 0, 100)
	if f.Doc != nil {
		layers = append(layers, nodeLayer(f.Doc, commentColor))
	}
	if f.Package.IsValid() {
		layers = append(layers, layer(f.Package, len("package"), keywordColor))
	}
	for _, importSpec := range f.Imports {
		layers = append(layers, nodeLayer(importSpec, stringColor))
	}
	for _, comment := range f.Comments {
		layers = append(layers, nodeLayer(comment, commentColor))
	}<commit_msg>Remove resolved TODO<commit_after>// This is free and unencumbered software released into the public
// domain.  For more information, see <http://unlicense.org> or the
// accompanying UNLICENSE file.

package syntax

import (
	"go/ast"
	"go/parser"
	"go/token"

	"github.com/nelsam/gxui"
)

func zeroBasedPos(pos token.Pos) uint64 {
	if pos < 1 {
		panic("Positions of 0 are not valid positions")
	}
	return uint64(pos) - 1
}

func nodeLayer(node ast.Node, colors ...gxui.Color) *gxui.CodeSyntaxLayer {
	return layer(node.Pos(), int(node.End()-node.Pos()), colors...)
}

func layer(pos token.Pos, length int, colors ...gxui.Color) *gxui.CodeSyntaxLayer {
	if length == 0 {
		return nil
	}
	if len(colors) == 0 {
		panic("No colors passed to layer()")
	}
	if len(colors) > 2 {
		panic("Only two colors (text and background) are currently supported")
	}
	layer := gxui.CreateCodeSyntaxLayer()
	layer.Add(int(zeroBasedPos(pos)), length)
	layer.SetColor(colors[0])
	if len(colors) > 1 {
		layer.SetBackgroundColor(colors[1])
	}
	return layer
}

func Layers(filename, text string) (gxui.CodeSyntaxLayers, error) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, filename, text, 0)
	layers := make(gxui.CodeSyntaxLayers, 0, 100)
	if f.Doc != nil {
		layers = append(layers, nodeLayer(f.Doc, commentColor))
	}
	if f.Package.IsValid() {
		layers = append(layers, layer(f.Package, len("package"), keywordColor))
	}
	for _, importSpec := range f.Imports {
		layers = append(layers, nodeLayer(importSpec, stringColor))
	}
	for _, comment := range f.Comments {
		layers = append(layers, nodeLayer(comment, commentColor))
	}
{'fraction_non_alphanumeric': 0.14389578915480158, 'fraction_numerical': 0.01090578612541654, 'mean_word_length': 4.308681672025723, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6135017', 'n_tokens_mistral': 1204, 'n_tokens_neox': 1147, 'n_words': 302}