<commit_before>package hamt32

import (
	"fmt"
	"log"
	"strings"

	"github.com/lleo/go-hamt-key"
)

type fullTable struct {
	hashPath key.HashVal30
	nents    uint
	nodes    [tableCapacity]nodeI
}

func (t fullTable) copy() tableI {
	var nt = new(fullTable)
	nt.hashPath = t.hashPath
	nt.nents = t.nents
	nt.nodes = t.nodes
	return nt
}

func createRootFullTable(lf leafI) tableI {
	var idx = lf.Hash30().Index(0)

	var ft = new(fullTable)
	//ft.hashPath = 0
	//ft.nents = 0
	ft.set(idx, lf)

	return ft
}

func createFullTable(depth uint, leaf1 leafI, leaf2 *flatLeaf) tableI {
	if depth < 1 {
		log.Panic("createFullTable(): depth < 1")
	}
	var hp1 = leaf1.Hash30() & key.HashPathMask30(depth-1)
	var hp2 = leaf2.Hash30() & key.HashPathMask30(depth-1)
	if hp1 != hp2 {
		log.Panic("newCompressedTable(): hp1,%s != hp2,%s",
			hp1.HashPathString(depth), hp2.HashPathString(depth))
	}
	//for d := uint(0); d < depth; d++ {
	//	if leaf1.Hash30().Index(d) != leaf2.Hash30().Index(d) {
	//		log.Panicf("createFullTable(): leaf1.Hash30().Index(%d) != leaf2.Hash30().Index(%d)", d, d)
	//	}
	//}

	var retTable = new(fullTable)
	//retTable.hashPath = leaf1.Hash30() & key.HashPathMask30(depth-1)
	retTable.hashPath = leaf1.Hash30().HashPath(depth)

	var idx1 = leaf1.Hash30().Index(depth)
	var idx2 = leaf2.Hash30().Index(depth)
	if idx1 != idx2 {
		retTable.insert(idx1, leaf1)
		retTable.insert(idx2, leaf2)
	} else { //idx1 == idx2
		var node nodeI
		if depth == maxDepth {
			node = newCollisionLeaf(append(leaf1.keyVals(), leaf2.keyVals()...))
		} else {
			node = createFullTable(depth+1, leaf1, leaf2)
		}
		retTable.insert(idx1, node)
	}

	return retTable
}

func upgradeToFullTable(hashPath key.HashVal30, ents []tableEntry) *fullTable {
	var ft = new(fullTable)
	ft.hashPath = hashPath
	ft.nents = uint(len(ents))

	for _, ent := range ents {
		ft.nodes[ent.idx] = ent.node
	}

	return ft
}

func (t *fullTable) Hash30() key.HashVal30 {
	return t.hashPath
}

func (t *fullTable) String() string {
	return fmt.Sprintf("fullTable{hashPath=%s, nentries()=%d}",
		t.hashPath, t.nentries())
}

func (t *fullTable) LongString(indent string, depth uint) string {
	var strs = make([]string, 3+len(t.nodes))

	strs[0] = indent + "fullTable{"
	strs[1] = indent + fmt.Sprintf("\tnents=%d,", t.nents)

	for i, n := range t.nodes {
		if t.nodes[i] == nil {
			strs[2+i] = indent + fmt.Sprintf("\tnodes[%d]: nil", i)
		} else {
			if t, isTable := t.nodes[i].(tableI); isTable {
				strs[2+i] = indent + fmt.Sprintf("\tnodes[%d]:\n", i) +
					t.LongString(indent+"\t", depth+1)
			} else {
				strs[2+i] = indent + fmt.Sprintf("\tnodes[%d]: %s", i, n)
			}
		}
	}

	strs[len(strs)-1] = indent + "}"

	return strings.Join(strs, "\n")
}

func (t *fullTable) nentries() uint {
	return t.nents<commit_msg>Updated LongString() to only output non-nil table entries. Added new depth field to the fullTable struct. Updated String() and LongString() to report this new field.<commit_after>package hamt32

import (
	"fmt"
	"log"
	"strings"

	"github.com/lleo/go-hamt-key"
)

type fullTable struct {
	hashPath key.HashVal30
	depth    uint
	nents    uint
	nodes    [tableCapacity]nodeI
}

func (t fullTable) copy() tableI {
	var nt = new(fullTable)
	nt.hashPath = t.hashPath
	nt.depth = t.depth
	nt.nents = t.nents
	nt.nodes = t.nodes
	return nt
}

func createRootFullTable(lf leafI) tableI {
	var idx = lf.Hash30().Index(0)

	var ft = new(fullTable)
	//ft.hashPath = 0
	//ft.depth = 0
	//ft.nents = 0
	ft.set(idx, lf)

	return ft
}

func createFullTable(depth uint, leaf1 leafI, leaf2 *flatLeaf) tableI {
	if depth < 1 {
		log.Panic("createFullTable(): depth < 1")
	}
	var hp1 = leaf1.Hash30() & key.HashPathMask30(depth-1)
	var hp2 = leaf2.Hash30() & key.HashPathMask30(depth-1)
	if hp1 != hp2 {
		log.Panic("newCompressedTable(): hp1,%s != hp2,%s",
			hp1.HashPathString(depth), hp2.HashPathString(depth))
	}
	//for d := uint(0); d < depth; d++ {
	//	if leaf1.Hash30().Index(d) != leaf2.Hash30().Index(d) {
	//		log.Panicf("createFullTable(): leaf1.Hash30().Index(%d) != leaf2.Hash30().Index(%d)", d, d)
	//	}
	//}

	var retTable = new(fullTable)
	//retTable.hashPath = leaf1.Hash30() & key.HashPathMask30(depth-1)
	retTable.hashPath = leaf1.Hash30().HashPath(depth)
	retTable.depth = depth

	var idx1 = leaf1.Hash30().Index(depth)
	var idx2 = leaf2.Hash30().Index(depth)
	if idx1 != idx2 {
		retTable.insert(idx1, leaf1)
		retTable.insert(idx2, leaf2)
	} else { //idx1 == idx2
		var node nodeI
		if depth == maxDepth {
			node = newCollisionLeaf(append(leaf1.keyVals(), leaf2.keyVals()...))
		} else {
			node = createFullTable(depth+1, leaf1, leaf2)
		}
		retTable.insert(idx1, node)
	}

	return retTable
}

func upgradeToFullTable(hashPath key.HashVal30, ents []tableEntry) *fullTable {
	var ft = new(fullTable)
	ft.hashPath = hashPath
	ft.nents = uint(len(ents))

	for _, ent := range ents {
		ft.nodes[ent.idx] = ent.node
	}

	return ft
}

func (t *fullTable) Hash30() key.HashVal30 {
	return t.hashPath
}

func (t *fullTable) String() string {
	return fmt.Sprintf("fullTable{hashPath=%s, depth=%d, nentries()=%d}",
		t.hashPath, t.depth, t.nentries())
}

func (t *fullTable) LongString(indent string, depth uint) string {
	var strs = make([]string, 3+t.nentries())

	strs[0] = indent + "fullTable{"
	strs[1] = indent + fmt.Sprintf("\thashPath=%s, depth=%d, nents=%d,",
		t.hashPath.HashPathString(depth+1), t.depth, t.nents)

	var j = 0
	for i, n := range t.nodes {
		if t.nodes[i] != nil {
			if t, isTable := t.nodes[i].(tableI); isTable {
				strs[2+j] = indent + fmt.Sprintf("\tnodes[%d]:\n", i) +
					t.LongString(indent+"\t", depth+1)
			} else {
				strs[2+j] = indent + fmt.Sprintf("\tnodes[%d]: %s", i, n)
			}
			j++
		}
	}

	strs[len(strs)-1] = indent + "}"

	return strings.Join(strs, "\n")
}

func (t *fullTable) nentries() uint {
	return t.nents
{'fraction_non_alphanumeric': 0.17493160054719561, 'fraction_numerical': 0.03146374829001368, 'mean_word_length': 3.7207425343018565, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7179883', 'n_tokens_mistral': 2547, 'n_tokens_neox': 2389, 'n_words': 509}