<commit_before>package tsdb

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"

	"github.com/cespare/xxhash"
	"github.com/influxdata/influxdb/models"
	"github.com/influxdata/influxdb/pkg/mmap"
	"github.com/influxdata/influxdb/pkg/rhh"
)

// ErrSeriesOverflow is returned when too many series are added to a series writer.
var ErrSeriesOverflow = errors.New("series overflow")

// SeriesIDSize is the size in bytes of a series key ID.
const SeriesIDSize = 8

// Series flag constants.
const (
	SeriesFileFlagSize      = 1
	SeriesFileTombstoneFlag = 0x01
)

// MaxSeriesFileHashSize is the maximum number of series in a single hash map.
const MaxSeriesFileHashSize = (1 << 20 * SeriesMapLoadFactor) / 100 // (1MB * 90) / 100 == ~943K

// SeriesMapThreshold is the number of series IDs to hold in the in-memory
// series map before compacting and rebuilding the on-disk representation.
const SeriesMapThreshold = 1 << 22 // ~4M ids * 8 bytes per id == ~32MB

// SeriesFile represents the section of the index that holds series data.
type SeriesFile struct {
	mu   sync.RWMutex
	path string
	data []byte
	file *os.File
	w    *bufio.Writer
	size int64

	seriesMap           *seriesMap
	compactingSeriesMap *seriesMap

	// MaxSize is the maximum size of the file.
	MaxSize int64
}

// NewSeriesFile returns a new instance of SeriesFile.<commit_msg>Update series map threshold<commit_after>package tsdb

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"

	"github.com/cespare/xxhash"
	"github.com/influxdata/influxdb/models"
	"github.com/influxdata/influxdb/pkg/mmap"
	"github.com/influxdata/influxdb/pkg/rhh"
)

// ErrSeriesOverflow is returned when too many series are added to a series writer.
var ErrSeriesOverflow = errors.New("series overflow")

// SeriesIDSize is the size in bytes of a series key ID.
const SeriesIDSize = 8

// Series flag constants.
const (
	SeriesFileFlagSize      = 1
	SeriesFileTombstoneFlag = 0x01
)

// MaxSeriesFileHashSize is the maximum number of series in a single hash map.
const MaxSeriesFileHashSize = (1 << 20 * SeriesMapLoadFactor) / 100 // (1MB * 90) / 100 == ~943K

// SeriesMapThreshold is the number of series IDs to hold in the in-memory
// series map before compacting and rebuilding the on-disk representation.
const SeriesMapThreshold = 1 << 25 // ~33M ids * 8 bytes per id == 256MB

// SeriesFile represents the section of the index that holds series data.
type SeriesFile struct {
	mu   sync.RWMutex
	path string
	data []byte
	file *os.File
	w    *bufio.Writer
	size int64

	seriesMap           *seriesMap
	compactingSeriesMap *seriesMap

	// MaxSize is the maximum size of the file.
	MaxSize int64
}

// NewSeriesFile returns a new instance of SeriesFile.
{'fraction_non_alphanumeric': 0.09097345132743363, 'fraction_numerical': 0.02265486725663717, 'mean_word_length': 3.6327868852459018, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5859096', 'n_tokens_mistral': 979, 'n_tokens_neox': 925, 'n_words': 316}