<commit_before>package main

import(
	"fmt"
	"bytes"
)

type Forward struct {
	tags []Tag
	sequence string
	filled bool
	trellis *Trellis
	initialState *InitialState
	transition *Transition
	emission *Emission
}

// NewViterb ...
func NewForward(tags []Tag, sequence string, i *InitialState, t *Transition, e *Emission) *Forward {
	trellis := NewTrellis(tags, len(sequence))
	v := Forward{tags, sequence, false, trellis, i, t, e}
	return &v
}

func (v *Forward) String() string {
	buffer := bytes.NewBufferString(fmt.Sprintf("Forward: '%s'\n", v.sequence))
	buffer.WriteString(v.trellis.String())
	return fmt.Sprintf(buffer.String())
}

// Result ...
func (v *Forward) Result(tag Tag, index int) *Result {
	if !v.filled {
		v.FillTrellis()
	}
	return (*v.trellis)[tag][index]
}

// FillTrellis ...
func (v *Forward) FillTrellis() {
	if v.filled {
		return
	}
	for i, _ := range v.sequence {
		v.FillColumn(i)
	}
	v.filled = true
}

// FillColumn ...
func (v *Forward) FillColumn(index int) {
	if v.filled {
		return
	}
	for _, tag := range v.tags {
		(*v.trellis)[tag][index] = v.P(tag, index)
	}
}

// P ...
func (v *Forward) P(tag Tag, index int) *Result {
	if index == 0 {
		value := v.sequence[index]
		pI := v.initialState.P(tag)
		pE := v.emission.P(tag, value)
		return &Result{"e", pI * pE}
	} else {
	  return v.SumP(tag, index)
	}
}

// SumP ...
func (v *Forward) SumP(tag Tag, index int) *Result {
	sumResult := &Result{"e", 0.0}
  value := v.sequence[index]
	for _, givenTag := range v.tags {
		prevResult := (*v.trellis)[givenTag][index - 1]
		pT := v.transition.P(tag, givenTag)
		pE := v.emission.P(tag, value)
		p := prevResult.probability * pT * pE
		sumResult.probability += p
	}
	return sumResult
}
<commit_msg>fix forward<commit_after>package main

import(
	"fmt"
	"bytes"
)

type Forward struct {
	tags []Tag
	sequence string
	filled bool
	trellis *Trellis
	initialState *InitialState
	transition *Transition
	emission *Emission
}

// NewViterb ...
func NewForward(tags []Tag, sequence string, i *InitialState, t *Transition, e *Emission) *Forward {
	trellis := NewTrellis(tags, len(sequence))
	v := Forward{tags, sequence, false, trellis, i, t, e}
	return &v
}

func (v *Forward) String() string {
	buffer := bytes.NewBufferString(fmt.Sprintf("Forward: '%s'\n", v.sequence))
	buffer.WriteString(v.trellis.String())
	return fmt.Sprintf(buffer.String())
}

// Result ...
func (v *Forward) Result(tag Tag, index int) *Result {
	if !v.filled {
		v.FillTrellis()
	}
	return (*v.trellis)[tag][index]
}

// FillTrellis ...
func (v *Forward) FillTrellis() {
	if v.filled {
		return
	}
	for i, _ := range v.sequence {
		v.FillColumn(i)
	}
	v.filled = true
}

// FillColumn ...
func (v *Forward) FillColumn(index int) {
	if v.filled {
		return
	}
	for _, tag := range v.tags {
		(*v.trellis)[tag][index] = v.P(tag, index)
	}
}

// P ...
func (v *Forward) P(tag Tag, index int) *Result {
	if index == 0 {
		value := v.sequence[index]
		pI := v.initialState.P(tag)
		pE := v.emission.P(tag, value)
		return &Result{"e", pI * pE}
	} else {
	  return v.SumP(tag, index)
	}
}

// SumP ...
func (v *Forward) SumP(tag Tag, index int) *Result {
  value := v.sequence[index]
	sumResult := &Result{"e", 0.0}
	for _, givenTag := range v.tags {
		prevResult := (*v.trellis)[givenTag][index - 1]
		p := prevResult.probability
		pT := v.transition.P(tag, givenTag)
		sumResult.probability += pT * p
	}
  pE := v.emission.P(tag, value)
	sumResult.probability = sumResult.probability * pE
	return sumResult
}

{'fraction_non_alphanumeric': 0.16975748930099857, 'fraction_numerical': 0.002282453637660485, 'mean_word_length': 3.553246753246753, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3126285', 'n_tokens_mistral': 1402, 'n_tokens_neox': 1354, 'n_words': 337}