<commit_before>package platform

import (
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"fmt"
)

const maxIDLength = 16

// ID is a unique identifier.
type ID struct {
	value *uint64
}

// IDGenerator represents a generator for IDs.
type IDGenerator interface {
	// ID creates unique byte slice ID.
	ID() ID
}

// IDFromString creates an ID from a given string
func IDFromString(idstr string) (*ID, error) {
	var id ID
	err := id.DecodeFromString(idstr)
	if err != nil {
		return nil, err
	}
	return &id, nil
}

// Empty tells whether the ID is empty or not.
func (i *ID) Empty() bool {
	return i.value == nil
}

// Decode parses b as a hex-encoded byte-slice-string.
func (i *ID) Decode(b []byte) error {
	if len(b) != maxIDLength {
		return fmt.Errorf("input must be an array of 8 bytes")
	}

	dst := make([]byte, hex.DecodedLen(maxIDLength))
	_, err := hex.Decode(dst, b)
	if err != nil {
		return err
	}
	out := binary.BigEndian.Uint64(dst)
	i.value = &out
	return nil
}

// DecodeFromString parses s as a hex-encoded string.
func (i *ID) DecodeFromString(s string) error {
	return i.Decode([]byte(s))
}

// Encode converts ID to a hex-encoded byte-slice-string.
func (i ID) Encode() ([]byte, error) {
	if i.Empty() {
		return nil, fmt.Errorf("cannot encode an empty ID")
	}
	b := make([]byte, hex.DecodedLen(maxIDLength))
	binary.BigEndian.PutUint64(b, *i.value)

	dst := make([]byte, hex.EncodedLen(len(b)))
	hex.Encode(dst, b)
	return dst, nil
}

// String returns the ID as a hex encoded string
func (i ID) String() string {
	res, err := i.Encode()
	if err != nil {
		return ""
	}
	return string(res)
}

// UnmarshalJSON implements JSON unmarshaller for IDs.
func (i *ID) UnmarshalJSON(b []byte) error {
	b = b[1 : len(b)-1]
	return i.Decode(b)
}

// MarshalJSON implements JSON marshaller for IDs.
func (i ID) MarshalJSON() ([]byte, error) {
	id, err := i.Encode()
	if err != nil {
		return nil, err
	}
	return json.Marshal(string(id[:]))
}<commit_msg>Switch ID back to uint64<commit_after>package platform

import (
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"fmt"
)

const maxIDLength = 16

// ID is a unique identifier.
type ID uint64

// IDGenerator represents a generator for IDs.
type IDGenerator interface {
	// ID creates unique byte slice ID.
	ID() ID
}

// IDFromString creates an ID from a given string.
func IDFromString(str string) (*ID, error) {
	var id ID
	err := id.DecodeFromString(str)
	if err != nil {
		return nil, err
	}
	return &id, nil
}

// Decode parses b as a hex-encoded byte-slice-string.
func (i *ID) Decode(b []byte) error {
	if len(b) != maxIDLength {
		return fmt.Errorf("input must be an array of 8 bytes")
	}

	dst := make([]byte, hex.DecodedLen(maxIDLength))
	_, err := hex.Decode(dst, b)
	if err != nil {
		return err
	}
	*i = ID(binary.BigEndian.Uint64(dst))
	return nil
}

// DecodeFromString parses s as a hex-encoded string.
func (i *ID) DecodeFromString(s string) error {
	return i.Decode([]byte(s))
}

// Encode converts ID to a hex-encoded byte-slice-string.
func (i ID) Encode() []byte {
	b := make([]byte, hex.DecodedLen(maxIDLength))
	binary.BigEndian.PutUint64(b, uint64(i))

	dst := make([]byte, hex.EncodedLen(len(b)))
	hex.Encode(dst, b)
	return dst
}

// String returns the ID as a hex encoded string
func (i ID) String() string {
	return string(i.Encode())
}

// UnmarshalJSON implements JSON unmarshaller for IDs.
func (i *ID) UnmarshalJSON(b []byte) error {
	b = b[1 : len(b)-1]
	return i.Decode(b)
}

// MarshalJSON implements JSON marshaller for IDs.
func (i ID) MarshalJSON() ([]byte, error) {
	return json.Marshal(string(i.Encode()))
}
{'fraction_non_alphanumeric': 0.14119922630560927, 'fraction_numerical': 0.007184305056645482, 'mean_word_length': 3.6891191709844557, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5050133', 'n_tokens_mistral': 1343, 'n_tokens_neox': 1291, 'n_words': 428}