<commit_before>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/couchbase/ghistogram"
)

// A collection implements the Collection interface.
type collection struct {
	options *CollectionOptions

	stopCh          chan struct{}
	pingMergerCh    chan ping
	doneMergerCh    chan struct{}
	donePersisterCh chan struct{}
	idleMergerTimer *time.Timer

	m sync.Mutex // Protects the fields that follow.

	// When ExecuteBatch() has pushed a new segment onto
	// stackDirtyTop, it can notify waiters like the merger via
	// waitDirtyIncomingCh (if non-nil).
	waitDirtyIncomingCh chan struct{}

	// When the persister has finished a persistence cycle, it can
	// notify waiters like the merger via waitDirtyOutgoingCh (if
	// non-nil).
	waitDirtyOutgoingCh chan struct{}

	// stats leverage sync/atomic counters.
	// All child collections simply point back to the same stats instance.
	// TODO: Have child collection specific stats.
	stats *CollectionStats

	// latestSnapshot caches the most recent collection snapshot to avoid
	// new snapshot creations in the absence of new mutations.
	latestSnapshot Snapshot

	// highestIncarNum is the highest descendant collection
	// incarnation number seen by this collection hierarchy.  It
	// monotonically increases every time a new child collection is
	// created and helps distinguish child collection recreations with
	// the same name.
	highestIncarNum uint64

	// ----------------------------------------

	// stackDirtyTopCond is used to wait for space in stackDirtyTop.
	stackDirtyTopCond *sync.Cond

	// stackDirtyBaseCond is used to wait for non-nil stackDirtyBase.
	stackDirtyBaseCond *sync.Cond

	// ----------------------------------------

	// ExecuteBatch() will push new segments onto stackDirtyTop if
	// there is space.
	stackDirtyTop *segmentStack

	// The merger goroutine asynchronously, atomically grabs all
	// segments from stackDirtyTop and atomically moves them into
	// stackDirtyMid.  The merger will also merge segments in
	// stackDirtyMid to keep its height low.
	stackDirtyMid *segmentStack

	// stackDirtyBase represents the segments currently being
	// persisted.  It is optionally populated by the merger when there
	// are merged segments ready for persistence.  Will be nil when
	// persistence is not being used.
	stackDirtyBase *segmentStack

	// stackClean represents the segments that have been optionally
	// persisted by the persister, and can now be safely evicted, as
	// the lowerLevelSnapshot will contain the entries from
	// stackClean.  Will be nil when persistence is not being used.
	stackClean *segmentStack

	// lowerLevelSnapshot provides an optional, lower-level storage
	// implementation, when using the Collection as a cache.
	lowerLevelSnapshot *SnapshotWrapper

	// histograms from collection operations
	histograms ghistogram.Histograms

	// incarNum is a unique incarnation number assigned to this child
	// collection at the time of its creation.  It helps process child
	// collection recreations and is zero in the top-level collection.
	incarNum uint64

	// Map of child collection by name.
	// TODO: Most of the fields of the child collections are nil, so
	// it might be lighter to use a dedicated struct instead of
	// reusing the collection struct.
	childCollections map[string]*collection
}

// ------------------------------------------------------

// Start kicks off required background gouroutines.
func (m *collection) Start() error {
	if !m.options.ReadOnly {
		// Kick off merger and persister only when not in Read-Only mode
		go m.runMerger()
		go m.runPersister()
	}
	return nil
}

// Close synchronously stops background goroutines.
func (m *collection) Close() error {
	m.fireEvent(EventKindCloseStart, 0)
	startTime := time.Now()
	defer func() {
		m.fireEvent(EventKindClose, time.Now().Sub(startTime))
	}()

	atomic.AddUint64(&m.stats.TotCloseBeg, 1)

	m.m.Lock()

	close(m.stopCh)

	m.stackDirtyTopCond.Broadcast()  // Awake all ExecuteBatch()'ers.
	m.stackDirtyBaseCond.Broadcast() // Awake persister.

	if !m.options.ReadOnly {
		m.m.Unlock()

		<-m.doneMergerCh
		atomic.AddUint64(&m.stats.TotCloseMergerDone, 1)

		<-m.donePersisterCh
		atomic.AddUint64(&m.stats.TotClosePersisterDone, 1)

		m.m.Lock()
	}
<commit_msg>Call invalidateLatestSnapshot during Close() of collection<commit_after>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/couchbase/ghistogram"
)

// A collection implements the Collection interface.
type collection struct {
	options *CollectionOptions

	stopCh          chan struct{}
	pingMergerCh    chan ping
	doneMergerCh    chan struct{}
	donePersisterCh chan struct{}
	idleMergerTimer *time.Timer

	m sync.Mutex // Protects the fields that follow.

	// When ExecuteBatch() has pushed a new segment onto
	// stackDirtyTop, it can notify waiters like the merger via
	// waitDirtyIncomingCh (if non-nil).
	waitDirtyIncomingCh chan struct{}

	// When the persister has finished a persistence cycle, it can
	// notify waiters like the merger via waitDirtyOutgoingCh (if
	// non-nil).
	waitDirtyOutgoingCh chan struct{}

	// stats leverage sync/atomic counters.
	// All child collections simply point back to the same stats instance.
	// TODO: Have child collection specific stats.
	stats *CollectionStats

	// latestSnapshot caches the most recent collection snapshot to avoid
	// new snapshot creations in the absence of new mutations.
	latestSnapshot Snapshot

	// highestIncarNum is the highest descendant collection
	// incarnation number seen by this collection hierarchy.  It
	// monotonically increases every time a new child collection is
	// created and helps distinguish child collection recreations with
	// the same name.
	highestIncarNum uint64

	// ----------------------------------------

	// stackDirtyTopCond is used to wait for space in stackDirtyTop.
	stackDirtyTopCond *sync.Cond

	// stackDirtyBaseCond is used to wait for non-nil stackDirtyBase.
	stackDirtyBaseCond *sync.Cond

	// ----------------------------------------

	// ExecuteBatch() will push new segments onto stackDirtyTop if
	// there is space.
	stackDirtyTop *segmentStack

	// The merger goroutine asynchronously, atomically grabs all
	// segments from stackDirtyTop and atomically moves them into
	// stackDirtyMid.  The merger will also merge segments in
	// stackDirtyMid to keep its height low.
	stackDirtyMid *segmentStack

	// stackDirtyBase represents the segments currently being
	// persisted.  It is optionally populated by the merger when there
	// are merged segments ready for persistence.  Will be nil when
	// persistence is not being used.
	stackDirtyBase *segmentStack

	// stackClean represents the segments that have been optionally
	// persisted by the persister, and can now be safely evicted, as
	// the lowerLevelSnapshot will contain the entries from
	// stackClean.  Will be nil when persistence is not being used.
	stackClean *segmentStack

	// lowerLevelSnapshot provides an optional, lower-level storage
	// implementation, when using the Collection as a cache.
	lowerLevelSnapshot *SnapshotWrapper

	// histograms from collection operations
	histograms ghistogram.Histograms

	// incarNum is a unique incarnation number assigned to this child
	// collection at the time of its creation.  It helps process child
	// collection recreations and is zero in the top-level collection.
	incarNum uint64

	// Map of child collection by name.
	// TODO: Most of the fields of the child collections are nil, so
	// it might be lighter to use a dedicated struct instead of
	// reusing the collection struct.
	childCollections map[string]*collection
}

// ------------------------------------------------------

// Start kicks off required background gouroutines.
func (m *collection) Start() error {
	if !m.options.ReadOnly {
		// Kick off merger and persister only when not in Read-Only mode
		go m.runMerger()
		go m.runPersister()
	}
	return nil
}

// Close synchronously stops background goroutines.
func (m *collection) Close() error {
	m.fireEvent(EventKindCloseStart, 0)
	startTime := time.Now()
	defer func() {
		m.fireEvent(EventKindClose, time.Now().Sub(startTime))
	}()

	atomic.AddUint64(&m.stats.TotCloseBeg, 1)

	m.m.Lock()

	m.invalidateLatestSnapshotLOCKED()

	close(m.stopCh)

	m.stackDirtyTopCond.Broadcast()  // Awake all ExecuteBatch()'ers.
	m.stackDirtyBaseCond.Broadcast() // Awake persister.

	if !m.options.ReadOnly {
		m.m.Unlock()

		<-m.doneMergerCh
		atomic.AddUint64(&m.stats.TotCloseMergerDone, 1)

		<-m.donePersisterCh
		atomic.AddUint64(&m.stats.TotClosePersisterDone, 1)

		m.m.Lock()
	}

{'fraction_non_alphanumeric': 0.10156886715566422, 'fraction_numerical': 0.004482477587612062, 'mean_word_length': 3.271975630983464, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6045455', 'n_tokens_mistral': 2878, 'n_tokens_neox': 2682, 'n_words': 1090}