<commit_before>package types

type Pagination struct {
	Start string        `json:"start"`
	End   string        `json:"end"`
	Chunk []interface{} `json:"chunk"`
}

type InitialSync struct {
	End      string          `json:"end"`
	Presence []PresenceEvent `json:"presence"`
	Rooms    []RoomSummary   `json:"rooms"`
}

type RoomSummary struct {
	Membership Membership `json:"membership"`
	RoomId     RoomId     `json:"room_id"`
	Messages   []Event    `json:"messages"`
	State      []State    `json:"state"`
}

type RoomInitialSync struct {
	RoomSummary
	Presence PresenceEvent `json:"presence"`
}<commit_msg>types/stream: added StreamToken type<commit_after>package types

import (
	"fmt"

	"github.com/Rugvip/bullettime/utils"
)

type Pagination struct {
	Start string        `json:"start"`
	End   string        `json:"end"`
	Chunk []interface{} `json:"chunk"`
}

type InitialSync struct {
	End      string          `json:"end"`
	Presence []PresenceEvent `json:"presence"`
	Rooms    []RoomSummary   `json:"rooms"`
}

type RoomSummary struct {
	Membership Membership `json:"membership"`
	RoomId     RoomId     `json:"room_id"`
	Messages   []Event    `json:"messages"`
	State      []State    `json:"state"`
}

type RoomInitialSync struct {
	RoomSummary
	Presence PresenceEvent `json:"presence"`
}

type StreamToken struct {
	MessageIndex  int64
	PresenceIndex int64
	TypingIndex   int64
}

type TokenParseError string

func (e TokenParseError) Error() string {
	return "failed to parse token: " + string(e)
}

func (t StreamToken) String() string {
	return fmt.Sprintf("s%d_%d_%d", t.MessageIndex, t.PresenceIndex, t.TypingIndex)
}

func ParseStreamToken(str string) (StreamToken, error) {
	var message, presence, typing int64
	count, err := fmt.Sscanf(str, "s%d_%d_%d", &message, &presence, &typing)
	if err != nil {
		return StreamToken{}, TokenParseError(err.Error())
	}
	if count != 3 {
		return StreamToken{}, TokenParseError("token does not match format")
	}
	return StreamToken{message, presence, typing}, nil
}

func (t *StreamToken) UnmarshalJSON(bytes []byte) (err error) {
	*t, err = ParseStreamToken(utils.StripQuotes(string(bytes)))
	return
}

func (t StreamToken) MarshalJSON() ([]byte, error) {
	return []byte(`"` + t.String() + `"`), nil
}
{'fraction_non_alphanumeric': 0.13543516873889877, 'fraction_numerical': 0.003996447602131439, 'mean_word_length': 4.288732394366197, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2590404', 'n_tokens_mistral': 749, 'n_tokens_neox': 736, 'n_words': 194}