<commit_before>package serf

import (
	"errors"
	"fmt"
	"github.com/hashicorp/memberlist"
	"log"
	"math/rand"
	"net"
	"os"
	"sync"
	"time"
)

func init() {
	// Seed the random number generator
	rand.Seed(time.Now().UnixNano())
}

// Serf is a single node that is part of a single cluster that gets
// events about joins/leaves/failures/etc. It is created with the Create
// method.
//
// All functions on the Serf structure are safe to call concurrently.
type Serf struct {
	broadcasts    *memberlist.TransmitLimitedQueue
	clock         LamportClock
	config        *Config
	failedMembers []*memberState
	leftMembers   []*memberState
	memberlist    *memberlist.Memberlist
	memberLock    sync.RWMutex
	members       map[string]*memberState

	stateLock  sync.Mutex
	state      SerfState
	shutdownCh chan struct{}
}

// SerfState is the state of the Serf instance.
type SerfState int

const (
	SerfAlive SerfState = iota
	SerfLeft
	SerfShutdown
)

// Member is a single member of the Serf cluster.
type Member struct {
	Name   string
	Addr   net.IP
	Role   string
	Status MemberStatus
}

// MemberStatus is the state that a member is in.
type MemberStatus int

const (
	StatusNone MemberStatus = iota
	StatusAlive
	StatusLeaving
	StatusLeft
	StatusFailed
)

// memberState is used to track members that are no longer active due to
// leaving, failing, partitioning, etc. It tracks the member along with
// when that member was marked as leaving.
type memberState struct {
	Member
	joinLTime LamportTime // lamport clock time of join
	leaveTime time.Time   // wall clock time of leave
}

// Create creates a new Serf instance, starting all the background tasks
// to maintain cluster membership information.
//
// After calling this function, the configuration should no longer be used
// or modified by the caller.
func Create(conf *Config) (*Serf, error) {
	if conf.NodeName == "" {
		// Default the node name to the hostname
		hostname, err := os.Hostname()
		if err != nil {
			return nil, fmt.Errorf("Error setting NodeName to hostname: %s", err)
		}

		conf.NodeName = hostname
	}

	if conf.BroadcastTimeout == 0 {
		// Set a cautious default for the timeout for leave broadcasts.
		conf.BroadcastTimeout = 5 * time.Second
	}

	if conf.LeaveTimeout == 0 {
		// Set a cautious default for a leave timeout.
		conf.LeaveTimeout = 120 * time.Second
	}

	if conf.ReapInterval == 0 {
		// Set a reasonable default for ReapInterval
		conf.ReapInterval = 15 * time.Second
	}

	if conf.ReconnectInterval == 0 {
		// Set a reasonable default for ReconnectInterval
		conf.ReconnectInterval = 30 * time.Second
	}

	if conf.ReconnectTimeout == 0 {
		// Set a reasonable default for ReconnectTimeout
		conf.ReconnectTimeout = 24 * time.Hour
	}

	if conf.TombstoneTimeout == 0 {
		// Set a reasonable default for TombstoneTimeout
		conf.TombstoneTimeout = 24 * time.Hour
	}

	if conf.QueueDepthWarning == 0 {
		// Set reasonable default for QueueDepthWarning
		conf.QueueDepthWarning = 128
	}

	if conf.MemberlistConfig == nil {
		conf.MemberlistConfig = memberlist.DefaultConfig()
	}

	serf := &Serf{
		config:     conf,
		members:    make(map[string]*memberState),
		shutdownCh: make(chan struct{}),
		state:      SerfAlive,
	}

	if conf.CoalescePeriod > 0 && conf.EventCh != nil {
		// Event coalescence is enabled, setup the channel.
		conf.EventCh = coalescedEventCh(conf.EventCh, serf.shutdownCh,
			conf.CoalescePeriod, conf.QuiescentPeriod)
	}

	// Setup the broadcast queue, which we use to send our own custom
	// broadcasts along the gossip channel.
	serf.broadcasts = &memberlist.TransmitLimitedQueue{
		NumNodes: func() int {
			serf.memberLock.RLock()
			defer serf.memberLock.RUnlock()
			return len(serf.members)
		},
		RetransmitMult: conf.MemberlistConfig.RetransmitMult,
	}

	// Modify the memberlist configuration with keys that we set
	//conf.MemberlistConfig.Events = &memberlist.ChannelEventDelegate{Ch: eventCh}
	conf.MemberlistConfig.Events = &eventDelegate{serf: serf}
	conf.MemberlistConfig.Delegate = &delegate{serf: serf}
	conf.MemberlistConfig.Name = conf.NodeName

	// Create the underlying memberlist that will manage membership
	// and failure detection for the Serf instance.
	memberlist, err := memberlist.Create(conf.MemberlistConfig)
	if err != nil {
		return nil, err
	}

	serf.memberlist = memberlist

	// Start the background tasks. See the documentation above each method
	// for more information on their role.
	go serf.handleReap()
	go serf.handleReconnect()

	return serf, nil
}

// Join joins an existing Serf cluster. Returns the number of nodes
// successfully contacted. The returned error will be non-nil only in the
// case that no nodes could be contacted.
func (s *Serf) Join(existing []string) (int, error) {
	s.stateLock.Lock()
	defer s.stateLock.Unlock()

	if s.state == SerfShutdown {
		panic("Serf can't Join after Shutdown")
	}

	return s.memberlist.Join(existing)
}

// Leave gracefully exits the cluster. It is safe to call this multiple
// times.
func (s *Serf) Leave() error {
	s.stateLock.Lock()
	defer s.stateLock.Unlock()

	if s.state == SerfLeft {
		return nil
	} else if s.state == SerfShutdown {
		panic("Leave after Shutdown")
	}

	// Construct the message for the graceful leave
	msg := messageLeave{Node: s.config.NodeName}

	// Process the leave locally
	s.handleNodeLeaveIntent(&msg)

	if len(s.members) > 1 {
		notifyCh := make(chan struct{})
		if err := s.broadcast(messageLeaveType, &msg, notifyCh); err != nil {
			return err
		}

		select {
		case <-notifyCh:
		case <-time.After(s.config.BroadcastTimeout):
			return errors.New("timeout while waiting for graceful leave")
		}
	}

	err := s.memberlist.Leave(s.config.BroadcastTimeout)
	if err != nil {
		return err
	}

	s.state = SerfLeft
	return nil
}

// Members returns a point-in-time snapshot of the members of this cluster.
func (s *Serf) Members() []Member {
	s.memberLock.RLock()
	defer s.memberLock.RUnlock()

	members := make([]Member, 0, len(s.members))
	for _, m := range s.members {
		members = append(members, m.Member)
	}

	return members
}

// RemoveFailedNode forcibly removes a failed node from the cluster
// immediately, instead of waiting for the reaper to eventually reclaim it.
func (s *Serf) RemoveFailedNode(node string) error {
	// Construct the message to broadcast
	msg := messageRemoveFailed{Node: node}

	// Process our own event
	s.handleNodeForceRemove(&msg)

	// If we have no members, then we don't need to broadcast
	s.memberLock.RLock()
	if len(s.members) <= 1 {
		s.memberLock.RUnlock()
		return nil
	}
	s.memberLock.RUnlock()

	// Broadcast the remove
	notifyCh := make(chan struct{})
	if err := s.broadcast(messageRemoveFailedType, &msg, notifyCh); err != nil {
		return err
	}

	// Wait for the broadcast
	select {
	case <-notifyCh:
	case <-time.After(s.config.BroadcastTimeout):
		return fmt.Errorf("timed out broadcasting node removal")
	}

	return nil
}

// Shutdown forcefully shuts down the Serf instance, stopping all network
// activity and background maintenance associated with the instance.
//
// This is not a graceful shutdown, and should be preceeded by a call
// to Leave. Otherwise, other nodes in the cluster will detect this node's
// exit as a node failure.
//
// It is safe to call this method multiple times.
func (s *Serf) Shutdown() error {
	s.stateLock.Lock()
	defer s.stateLock.Unlock()

	if s.state == SerfShutdown {
		return nil
	}

	if s.state != SerfLeft {
		log.Println("[WARN] Shutdown without a Leave")
	}

	err := s.memberlist.Shutdown()
	if err != nil {
		return err
	}

	s.state = SerfShutdown
	close(s.shutdownCh)
	return nil
}

// State is the current state of this Serf instance.
func (s *Serf) State() SerfState {
	s.stateLock.Lock()
	defer s.stateLock.Unlock()
	return s.state
}

// broadcast takes a Serf message type, encodes it for the wire, and queues
// the broadcast. If a notify channel is given, this channel will be closed
// when the broadcast is sent.
func (s *Serf) broadcast(t messageType, msg interface{}, notify chan<- struct{}) error {
	raw, err := encodeMessage(t, msg)
	if err != nil {
		return err
	}

	s.broadcasts.QueueBroadcast(&broadcast{
		msg:    raw,
		notify: notify,
	})

	return nil
}

// handleNodeForceRemove is invoked when we get a messageRemoveFailed
// message.
func (s *Serf) handleNodeForceRemove(remove *messageRemoveFailed) bool {
	s.memberLock.Lock()
	defer s.memberLock.Unlock()

	member, ok := s.members[remove.Node]
	if !ok {
		return false
	}

	// If the node isn't failed, then do nothing
	if member.Status != StatusFailed {
		return false
	}

	// Update the status to left
	member.Status = StatusLeft

	// Remove from the failed list and add to the left list. We add
	// to the left list so that when we do a sync, other nodes will
	// remove it from their failed list.
	s.failedMembers = removeOldMember(s.failedMembers, member.Name)
	s.leftMembers = append(s.leftMembers, member)
	return true
}

// handleNodeJoin is called when a node join event is received
// from memberlist.
func (s *Serf) handleNodeJoin(n *memberlist.Node) {
	s.memberLock.Lock()
	defer s.memberLock.Unlock()

	oldStatus := StatusNone
	member, ok := s.members[n.Name]
	if !ok {
		member = &memberState{
			Member: Member{
				Name: n.Name,
				Addr: net.IP(n.Addr),
				Role: string(n.Meta),
			},
		}

		s.members[n.Name] = member
	} else {
		oldStatus = member.Status
	}

	member.Status = StatusAlive
	member.leaveTime = time.Time{}

	// If node was previously in a failed state, then clean up some
	// internal accounting.
	if oldStatus == StatusFailed {
		s.failedMembers = removeOldMember(s.failedMembers, member.Name)
		s.leftMembers = removeOldMember(s.leftMembers, member.Name)
	}

	// Send an event along
	if s.config.EventCh != nil {
		s.config.EventCh <- Event{
			Type:    EventMemberJoin,
			Members: []Member{member.Member},
		}
	}
}

// handleNodeLeave is called when a node leave event is received
// from memberlist.
func (s *Serf) handleNodeLeave(n *memberlist.Node) {
	s.memberLock.Lock()
	defer s.memberLock.Unlock()

	member, ok := s.members[n.Name]
	if !ok {
		// We've never even heard of this node that is supposedly
		// leaving. Just ignore it completely.
		return
	}

	switch member.Status {
	case StatusLeaving:
		member.Status = StatusLeft
		member.leaveTime = time.Now()
		s.leftMembers = append(s.leftMembers, member)
	case StatusAlive:
		member.Status = StatusFailed
		member.leaveTime = time.Now()
		s.failedMembers = append(s.failedMembers, member)
	default:
		// Unknown state that it was in? Just don't do anything
		log.Printf("[WARN] Bad state when leave: %d", member.Status)
		return
	}

	// Send an event along
	if s.config.EventCh != nil {
		event := EventMemberLeave
		if member.Status != StatusLeft {
			event = EventMemberFailed
		}

		s.config.EventCh <- Event{
			Type:    event,
			Members: []Member{member.Member},
		}
	}
}

// handleNodeLeaveIntent is called when an intent to leave is received.
func (s *Serf) handleNodeLeaveIntent(leaveMsg *messageLeave) bool {
	s.memberLock.Lock()
	defer s.memberLock.Unlock()

	member, ok := s.members[leaveMsg.Node]
	if !ok {
		// We don't know this member so don't rebroadcast.
		return false
	}

	// If the node isn't alive, then this message is irrelevent and
	// we skip it.
	if member.Status != StatusAlive {
		return false
	}

	member.Status = StatusLeaving
<commit_msg>serf: provide the LTime and witness other LTime values<commit_after>package serf

import (
	"errors"
	"fmt"
	"github.com/hashicorp/memberlist"
	"log"
	"math/rand"
	"net"
	"os"
	"sync"
	"time"
)

func init() {
	// Seed the random number generator
	rand.Seed(time.Now().UnixNano())
}

// Serf is a single node that is part of a single cluster that gets
// events about joins/leaves/failures/etc. It is created with the Create
// method.
//
// All functions on the Serf structure are safe to call concurrently.
type Serf struct {
	broadcasts    *memberlist.TransmitLimitedQueue
	clock         LamportClock
	config        *Config
	failedMembers []*memberState
	leftMembers   []*memberState
	memberlist    *memberlist.Memberlist
	memberLock    sync.RWMutex
	members       map[string]*memberState

	stateLock  sync.Mutex
	state      SerfState
	shutdownCh chan struct{}
}

// SerfState is the state of the Serf instance.
type SerfState int

const (
	SerfAlive SerfState = iota
	SerfLeft
	SerfShutdown
)

// Member is a single member of the Serf cluster.
type Member struct {
	Name   string
	Addr   net.IP
	Role   string
	Status MemberStatus
}

// MemberStatus is the state that a member is in.
type MemberStatus int

const (
	StatusNone MemberStatus = iota
	StatusAlive
	StatusLeaving
	StatusLeft
	StatusFailed
)

// memberState is used to track members that are no longer active due to
// leaving, failing, partitioning, etc. It tracks the member along with
// when that member was marked as leaving.
type memberState struct {
	Member
	joinLTime LamportTime // lamport clock time of join
	leaveTime time.Time   // wall clock time of leave
}

// Create creates a new Serf instance, starting all the background tasks
// to maintain cluster membership information.
//
// After calling this function, the configuration should no longer be used
// or modified by the caller.
func Create(conf *Config) (*Serf, error) {
	if conf.NodeName == "" {
		// Default the node name to the hostname
		hostname, err := os.Hostname()
		if err != nil {
			return nil, fmt.Errorf("Error setting NodeName to hostname: %s", err)
		}

		conf.NodeName = hostname
	}

	if conf.BroadcastTimeout == 0 {
		// Set a cautious default for the timeout for leave broadcasts.
		conf.BroadcastTimeout = 5 * time.Second
	}

	if conf.LeaveTimeout == 0 {
		// Set a cautious default for a leave timeout.
		conf.LeaveTimeout = 120 * time.Second
	}

	if conf.ReapInterval == 0 {
		// Set a reasonable default for ReapInterval
		conf.ReapInterval = 15 * time.Second
	}

	if conf.ReconnectInterval == 0 {
		// Set a reasonable default for ReconnectInterval
		conf.ReconnectInterval = 30 * time.Second
	}

	if conf.ReconnectTimeout == 0 {
		// Set a reasonable default for ReconnectTimeout
		conf.ReconnectTimeout = 24 * time.Hour
	}

	if conf.TombstoneTimeout == 0 {
		// Set a reasonable default for TombstoneTimeout
		conf.TombstoneTimeout = 24 * time.Hour
	}

	if conf.QueueDepthWarning == 0 {
		// Set reasonable default for QueueDepthWarning
		conf.QueueDepthWarning = 128
	}

	if conf.MemberlistConfig == nil {
		conf.MemberlistConfig = memberlist.DefaultConfig()
	}

	serf := &Serf{
		config:     conf,
		members:    make(map[string]*memberState),
		shutdownCh: make(chan struct{}),
		state:      SerfAlive,
	}

	if conf.CoalescePeriod > 0 && conf.EventCh != nil {
		// Event coalescence is enabled, setup the channel.
		conf.EventCh = coalescedEventCh(conf.EventCh, serf.shutdownCh,
			conf.CoalescePeriod, conf.QuiescentPeriod)
	}

	// Setup the broadcast queue, which we use to send our own custom
	// broadcasts along the gossip channel.
	serf.broadcasts = &memberlist.TransmitLimitedQueue{
		NumNodes: func() int {
			serf.memberLock.RLock()
			defer serf.memberLock.RUnlock()
			return len(serf.members)
		},
		RetransmitMult: conf.MemberlistConfig.RetransmitMult,
	}

	// Modify the memberlist configuration with keys that we set
	//conf.MemberlistConfig.Events = &memberlist.ChannelEventDelegate{Ch: eventCh}
	conf.MemberlistConfig.Events = &eventDelegate{serf: serf}
	conf.MemberlistConfig.Delegate = &delegate{serf: serf}
	conf.MemberlistConfig.Name = conf.NodeName

	// Create the underlying memberlist that will manage membership
	// and failure detection for the Serf instance.
	memberlist, err := memberlist.Create(conf.MemberlistConfig)
	if err != nil {
		return nil, err
	}

	serf.memberlist = memberlist

	// Start the background tasks. See the documentation above each method
	// for more information on their role.
	go serf.handleReap()
	go serf.handleReconnect()

	return serf, nil
}

// Join joins an existing Serf cluster. Returns the number of nodes
// successfully contacted. The returned error will be non-nil only in the
// case that no nodes could be contacted.
func (s *Serf) Join(existing []string) (int, error) {
	s.stateLock.Lock()
	defer s.stateLock.Unlock()

	if s.state == SerfShutdown {
		panic("Serf can't Join after Shutdown")
	}

	return s.memberlist.Join(existing)
}

// Leave gracefully exits the cluster. It is safe to call this multiple
// times.
func (s *Serf) Leave() error {
	s.stateLock.Lock()
	defer s.stateLock.Unlock()

	if s.state == SerfLeft {
		return nil
	} else if s.state == SerfShutdown {
		panic("Leave after Shutdown")
	}

	// Construct the message for the graceful leave
	msg := messageLeave{
		LTime: s.clock.Increment(),
		Node:  s.config.NodeName,
	}

	// Process the leave locally
	s.handleNodeLeaveIntent(&msg)

	if len(s.members) > 1 {
		notifyCh := make(chan struct{})
		if err := s.broadcast(messageLeaveType, &msg, notifyCh); err != nil {
			return err
		}

		select {
		case <-notifyCh:
		case <-time.After(s.config.BroadcastTimeout):
			return errors.New("timeout while waiting for graceful leave")
		}
	}

	err := s.memberlist.Leave(s.config.BroadcastTimeout)
	if err != nil {
		return err
	}

	s.state = SerfLeft
	return nil
}

// Members returns a point-in-time snapshot of the members of this cluster.
func (s *Serf) Members() []Member {
	s.memberLock.RLock()
	defer s.memberLock.RUnlock()

	members := make([]Member, 0, len(s.members))
	for _, m := range s.members {
		members = append(members, m.Member)
	}

	return members
}

// RemoveFailedNode forcibly removes a failed node from the cluster
// immediately, instead of waiting for the reaper to eventually reclaim it.
func (s *Serf) RemoveFailedNode(node string) error {
	// Construct the message to broadcast
	msg := messageRemoveFailed{
		LTime: s.clock.Increment(),
		Node:  node,
	}

	// Process our own event
	s.handleNodeForceRemove(&msg)

	// If we have no members, then we don't need to broadcast
	s.memberLock.RLock()
	if len(s.members) <= 1 {
		s.memberLock.RUnlock()
		return nil
	}
	s.memberLock.RUnlock()

	// Broadcast the remove
	notifyCh := make(chan struct{})
	if err := s.broadcast(messageRemoveFailedType, &msg, notifyCh); err != nil {
		return err
	}

	// Wait for the broadcast
	select {
	case <-notifyCh:
	case <-time.After(s.config.BroadcastTimeout):
		return fmt.Errorf("timed out broadcasting node removal")
	}

	return nil
}

// Shutdown forcefully shuts down the Serf instance, stopping all network
// activity and background maintenance associated with the instance.
//
// This is not a graceful shutdown, and should be preceeded by a call
// to Leave. Otherwise, other nodes in the cluster will detect this node's
// exit as a node failure.
//
// It is safe to call this method multiple times.
func (s *Serf) Shutdown() error {
	s.stateLock.Lock()
	defer s.stateLock.Unlock()

	if s.state == SerfShutdown {
		return nil
	}

	if s.state != SerfLeft {
		log.Println("[WARN] Shutdown without a Leave")
	}

	err := s.memberlist.Shutdown()
	if err != nil {
		return err
	}

	s.state = SerfShutdown
	close(s.shutdownCh)
	return nil
}

// State is the current state of this Serf instance.
func (s *Serf) State() SerfState {
	s.stateLock.Lock()
	defer s.stateLock.Unlock()
	return s.state
}

// broadcast takes a Serf message type, encodes it for the wire, and queues
// the broadcast. If a notify channel is given, this channel will be closed
// when the broadcast is sent.
func (s *Serf) broadcast(t messageType, msg interface{}, notify chan<- struct{}) error {
	raw, err := encodeMessage(t, msg)
	if err != nil {
		return err
	}

	s.broadcasts.QueueBroadcast(&broadcast{
		msg:    raw,
		notify: notify,
	})

	return nil
}

// handleNodeForceRemove is invoked when we get a messageRemoveFailed
// message.
func (s *Serf) handleNodeForceRemove(remove *messageRemoveFailed) bool {
	// Witness a potentially newer time
	s.clock.Witness(remove.LTime)

	s.memberLock.Lock()
	defer s.memberLock.Unlock()

	member, ok := s.members[remove.Node]
	if !ok {
		return false
	}

	// If the node isn't failed, then do nothing
	if member.Status != StatusFailed {
		return false
	}

	// Update the status to left
	member.Status = StatusLeft

	// Remove from the failed list and add to the left list. We add
	// to the left list so that when we do a sync, other nodes will
	// remove it from their failed list.
	s.failedMembers = removeOldMember(s.failedMembers, member.Name)
	s.leftMembers = append(s.leftMembers, member)
	return true
}

// handleNodeJoin is called when a node join event is received
// from memberlist.
func (s *Serf) handleNodeJoin(n *memberlist.Node) {
	s.memberLock.Lock()
	defer s.memberLock.Unlock()

	oldStatus := StatusNone
	member, ok := s.members[n.Name]
	if !ok {
		member = &memberState{
			Member: Member{
				Name: n.Name,
				Addr: net.IP(n.Addr),
				Role: string(n.Meta),
			},
		}

		s.members[n.Name] = member
	} else {
		oldStatus = member.Status
	}

	member.Status = StatusAlive
	member.leaveTime = time.Time{}

	// If node was previously in a failed state, then clean up some
	// internal accounting.
	if oldStatus == StatusFailed {
		s.failedMembers = removeOldMember(s.failedMembers, member.Name)
		s.leftMembers = removeOldMember(s.leftMembers, member.Name)
	}

	// Send an event along
	if s.config.EventCh != nil {
		s.config.EventCh <- Event{
			Type:    EventMemberJoin,
			Members: []Member{member.Member},
		}
	}
}

// handleNodeLeave is called when a node leave event is received
// from memberlist.
func (s *Serf) handleNodeLeave(n *memberlist.Node) {
	s.memberLock.Lock()
	defer s.memberLock.Unlock()

	member, ok := s.members[n.Name]
	if !ok {
		// We've never even heard of this node that is supposedly
		// leaving. Just ignore it completely.
		return
	}

	switch member.Status {
	case StatusLeaving:
		member.Status = StatusLeft
		member.leaveTime = time.Now()
		s.leftMembers = append(s.leftMembers, member)
	case StatusAlive:
		member.Status = StatusFailed
		member.leaveTime = time.Now()
		s.failedMembers = append(s.failedMembers, member)
	default:
		// Unknown state that it was in? Just don't do anything
		log.Printf("[WARN] Bad state when leave: %d", member.Status)
		return
	}

	// Send an event along
	if s.config.EventCh != nil {
		event := EventMemberLeave
		if member.Status != StatusLeft {
			event = EventMemberFailed
		}

		s.config.EventCh <- Event{
			Type:    event,
			Members: []Member{member.Member},
		}
	}
}

// handleNodeLeaveIntent is called when an intent to leave is received.
func (s *Serf) handleNodeLeaveIntent(leaveMsg *messageLeave) bool {
	// Witness a potentially newer time
	s.clock.Witness(leaveMsg.LTime)

	s.memberLock.Lock()
	defer s.memberLock.Unlock()

	member, ok := s.members[leaveMsg.Node]
	if !ok {
		// We don't know this member so don't rebroadcast.
		return false
	}

	// If the node isn't alive, then this message is irrelevent and
	// we skip it.
	if member.Status != StatusAlive {
		return false
	}

	member.Status = StatusLeaving

{'fraction_non_alphanumeric': 0.10261202138293711, 'fraction_numerical': 0.002259985223173541, 'mean_word_length': 3.50381679389313, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3337418', 'n_tokens_mistral': 7758, 'n_tokens_neox': 7259, 'n_words': 2456}