<commit_before>package utils

import (
	"bytes"
	"encoding/json"
	"errors"

	"github.com/google/uuid"
	"github.com/lithammer/shortuuid"
)

// ID is unique primary key
type ID [18]byte

// NewID is new ID
func NewID(prefix byte) ID {
	ret := ID{}
	id := uuid.New()
	ret[0] = prefix
	ret[1] = '-'
	for i, v := range id {
		ret[2+i] = v
	}
	return ret
}

var newPrefix = []byte{0, 0}

// IsNew is New ID
func (id ID) IsNew() bool {
	return bytes.HasPrefix(id[:], newPrefix)
}

// String is ID to string
func (id ID) String() string {
	if id.IsNew() {
		return ""
	}
	var u uuid.UUID
	for i := 2; i < 18; i++ {
		u[i-2] = id[i]
	}
	uuidStr := shortuuid.DefaultEncoder.Encode(u)
	ret := make([]byte, len(uuidStr)+2)
	ret[0] = id[0]
	ret[1] = id[1]
	for i, v := range []byte(uuidStr) {
		ret[i+2] = v
	}
	return string(ret)
}

// Equal returns a boolean reporting whether id and other
func (id ID) Equal(other ID) bool {
	return bytes.Equal(id[:], other[:])<commit_msg>refactor: id IsNew<commit_after>package utils

import (
	"bytes"
	"encoding/json"
	"errors"

	"github.com/google/uuid"
	"github.com/lithammer/shortuuid"
)

// ID is unique primary key
type ID [18]byte

// NewID is new ID
func NewID(prefix byte) ID {
	ret := ID{}
	id := uuid.New()
	ret[0] = prefix
	ret[1] = '-'
	for i, v := range id {
		ret[2+i] = v
	}
	return ret
}

// IsNew is New ID
func (id ID) IsNew() bool {
	return id[1] != '-'
}

// String is ID to string
func (id ID) String() string {
	if id.IsNew() {
		return ""
	}
	var u uuid.UUID
	for i := 2; i < 18; i++ {
		u[i-2] = id[i]
	}
	uuidStr := shortuuid.DefaultEncoder.Encode(u)
	ret := make([]byte, len(uuidStr)+2)
	ret[0] = id[0]
	ret[1] = id[1]
	for i, v := range []byte(uuidStr) {
		ret[i+2] = v
	}
	return string(ret)
}

// Equal returns a boolean reporting whether id and other
func (id ID) Equal(other ID) bool {
	return bytes.Equal(id[:], other[:])
{'fraction_non_alphanumeric': 0.17712765957446808, 'fraction_numerical': 0.017553191489361703, 'mean_word_length': 3.2364864864864864, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6895689', 'n_tokens_mistral': 808, 'n_tokens_neox': 792, 'n_words': 228}