<commit_before>/*
 * Copyright 2017 <NAME>, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package table

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"

	"github.com/dgraph-io/badger/y"
	"github.com/pkg/errors"
)

const filePrefix = "table_"

const (
	Nothing = iota
	MemoryMap
	LoadToRAM
)

type keyOffset struct {
	key    []byte
	offset int
	len    int
}

type Table struct {
	sync.Mutex

	fd        *os.File // Own fd.
	tableSize int      // Initialized in OpenTable, using fd.Stat().

	blockIndex []keyOffset
	metadata   []byte
	ref        int32 // For file garbage collection.

	mapTableTo int
	mmap       []byte // Memory mapped.

	// The following are initialized once and const.
	smallest, biggest []byte // Smallest and largest keys.
	id                uint64
}

func (s *Table) IncrRef() {
	atomic.AddInt32(&s.ref, 1)
}

func (s *Table) DecrRef() {
	newRef := atomic.AddInt32(&s.ref, -1)
	if newRef == 0 {
		// We can safely delete this file, because for all the current files, we always have
		// at least one reference pointing to them.
		filename := s.fd.Name()
		y.Check(s.fd.Close())
		os.Remove(filename)
	}
}

type Block struct {
	offset int
	data   []byte
}

func (b Block) NewIterator() *BlockIterator {
	return &BlockIterator{data: b.data}
}

type byKey []keyOffset

func (b byKey) Len() int               { return len(b) }
func (b byKey) Swap(i int, j int)      { b[i], b[j] = b[j], b[i] }
func (b byKey) Less(i int, j int) bool { return bytes.Compare(b[i].key, b[j].key) < 0 }

// OpenTable assumes file has only one table and opens it.
func OpenTable(fd *os.File, mapTableTo int) (*Table, error) {
	id, ok := ParseFileID(fd.Name())
	if !ok {
		return nil, y.Errorf("Invalid filename: %s", fd.Name())
	}
	t := &Table{
		fd:         fd,
		ref:        1, // Caller is given one reference.
		id:         id,
		mapTableTo: mapTableTo,
	}
	fileInfo, err := fd.Stat()
	if err != nil {
		return nil, err
	}
	t.tableSize = int(fileInfo.Size())

	if mapTableTo == MemoryMap {
		//		t.mmap, err = syscall.Mmap(int(fd.Fd()), 0, int(fileInfo.Size()),
		//			syscall.PROT_READ, syscall.MAP_PRIVATE|syscall.MAP_POPULATE)
		t.mmap, err = syscall.Mmap(int(fd.Fd()), 0, int(fileInfo.Size()),
			syscall.PROT_READ, syscall.MAP_PRIVATE)
		if err != nil {
			y.Fatalf("Unable to map file: %v", err)
		}
	} else if mapTableTo == LoadToRAM {
		t.mmap = make([]byte, t.tableSize)
		read, err := t.fd.ReadAt(t.mmap, 0)<commit_msg>commit<commit_after>/*
 * Copyright 2017 <NAME>, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package table

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"

	"github.com/dgraph-io/badger/y"
	"github.com/pkg/errors"
)

const filePrefix = "table_"

const (
	Nothing = iota
	MemoryMap
	LoadToRAM
)

type keyOffset struct {
	key    []byte
	offset int
	len    int
}

type Table struct {
	sync.Mutex

	fd        *os.File // Own fd.
	tableSize int      // Initialized in OpenTable, using fd.Stat().

	blockIndex []keyOffset
	metadata   []byte
	ref        int32 // For file garbage collection.

	mapTableTo int
	mmap       []byte // Memory mapped.

	// The following are initialized once and const.
	smallest, biggest []byte // Smallest and largest keys.
	id                uint64
}

func (s *Table) IncrRef() {
	atomic.AddInt32(&s.ref, 1)
}

func (s *Table) DecrRef() {
	newRef := atomic.AddInt32(&s.ref, -1)
	if newRef == 0 {
		// We can safely delete this file, because for all the current files, we always have
		// at least one reference pointing to them.
		filename := s.fd.Name()
		y.Check(s.fd.Close())
		os.Remove(filename)
	}
}

type Block struct {
	offset int
	data   []byte
}

func (b Block) NewIterator() *BlockIterator {
	return &BlockIterator{data: b.data}
}

type byKey []keyOffset

func (b byKey) Len() int               { return len(b) }
func (b byKey) Swap(i int, j int)      { b[i], b[j] = b[j], b[i] }
func (b byKey) Less(i int, j int) bool { return bytes.Compare(b[i].key, b[j].key) < 0 }

// OpenTable assumes file has only one table and opens it.
func OpenTable(fd *os.File, mapTableTo int) (*Table, error) {
	id, ok := ParseFileID(fd.Name())
	if !ok {
		return nil, y.Errorf("Invalid filename: %s", fd.Name())
	}
	t := &Table{
		fd:         fd,
		ref:        1, // Caller is given one reference.
		id:         id,
		mapTableTo: mapTableTo,
	}
	fileInfo, err := fd.Stat()
	if err != nil {
		return nil, err
	}
	t.tableSize = int(fileInfo.Size())

	if mapTableTo == MemoryMap {
		t.mmap, err = syscall.Mmap(int(fd.Fd()), 0, int(fileInfo.Size()),
			syscall.PROT_READ, syscall.MAP_PRIVATE|syscall.MAP_POPULATE)
		//		t.mmap, err = syscall.Mmap(int(fd.Fd()), 0, int(fileInfo.Size()),
		//			syscall.PROT_READ, syscall.MAP_PRIVATE)
		if err != nil {
			y.Fatalf("Unable to map file: %v", err)
		}
	} else if mapTableTo == LoadToRAM {
		t.mmap = make([]byte, t.tableSize)
		read, err := t.fd.ReadAt(t.mmap, 0)
{'fraction_non_alphanumeric': 0.13385697921780396, 'fraction_numerical': 0.007854688267059401, 'mean_word_length': 3.0611295681063124, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7269971', 'n_tokens_mistral': 2273, 'n_tokens_neox': 2145, 'n_words': 693}