<commit_before>package transition

import (
	"bytes"
	"fmt"
	"log"
	"strings"
	"text/tabwriter"
	. "yap/alg/featurevector"
	"yap/util"
)

var IDLE = &TypedTransition{'I', 0}

type FeaturesList struct {
	Features   []Feature
	Transition Transition
	Previous   *FeaturesList
}

func (l *FeaturesList) String() string {
	var (
		retval []string      = make([]string, 0, 100)
		cur    *FeaturesList = l
	)
	for cur != nil {
		retval = append(retval, fmt.Sprintf("%v", cur.Transition))
		for _, val := range cur.Features {
			retval = append(retval, fmt.Sprintf("\t%v", val))
		}
		cur = cur.Previous
	}
	return strings.Join(retval, "\n")
}

type Transition interface {
	Type() byte
	Value() int
	Equal(other Transition) bool
}

type ConstTransition int

func (b ConstTransition) Type() byte {
	return '-'
}

func (b ConstTransition) Value() int {
	return int(b)
}

func (b ConstTransition) Equal(other Transition) bool {
	return b.Type() == other.Type() && b.Value() == other.Value()
}

type TypedTransition struct {
	T byte
	V int
}

func (t *TypedTransition) Type() byte {
	return t.T
}

func (t *TypedTransition) Value() int {
	return t.V
}

func (t *TypedTransition) Equal(other Transition) bool {
	log.Println("Comparing transitions", t, "=", other, "? is ", t.Type() == other.Type() && t.Value() == other.Value())
	return t.Type() == other.Type() && t.Value() == other.Value()
}

var _ Transition = ConstTransition(0)
var _ Transition = &TypedTransition{}
<commit_msg>Removes debug output<commit_after>package transition

import (
	"bytes"
	"fmt"
	// "log"
	"strings"
	"text/tabwriter"
	. "yap/alg/featurevector"
	"yap/util"
)

var IDLE = &TypedTransition{'I', 0}

type FeaturesList struct {
	Features   []Feature
	Transition Transition
	Previous   *FeaturesList
}

func (l *FeaturesList) String() string {
	var (
		retval []string      = make([]string, 0, 100)
		cur    *FeaturesList = l
	)
	for cur != nil {
		retval = append(retval, fmt.Sprintf("%v", cur.Transition))
		for _, val := range cur.Features {
			retval = append(retval, fmt.Sprintf("\t%v", val))
		}
		cur = cur.Previous
	}
	return strings.Join(retval, "\n")
}

type Transition interface {
	Type() byte
	Value() int
	Equal(other Transition) bool
}

type ConstTransition int

func (b ConstTransition) Type() byte {
	return '-'
}

func (b ConstTransition) Value() int {
	return int(b)
}

func (b ConstTransition) Equal(other Transition) bool {
	return b.Type() == other.Type() && b.Value() == other.Value()
}

type TypedTransition struct {
	T byte
	V int
}

func (t *TypedTransition) Type() byte {
	return t.T
}

func (t *TypedTransition) Value() int {
	return t.V
}

func (t *TypedTransition) Equal(other Transition) bool {
	// log.Println("Comparing transitions", t, "=", other, "? is ", t.Type() == other.Type() && t.Value() == other.Value())
	return t.Type() == other.Type() && t.Value() == other.Value()
}

var _ Transition = ConstTransition(0)
var _ Transition = &TypedTransition{}

{'fraction_non_alphanumeric': 0.15848289874703692, 'fraction_numerical': 0.004063664070436844, 'mean_word_length': 3.9647058823529413, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5497063', 'n_tokens_mistral': 1054, 'n_tokens_neox': 1019, 'n_words': 284}