<commit_before>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bolt

import (
	"bytes"
	"crypto/sha1"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"hash"

	"github.com/barakmich/glog"
	"github.com/boltdb/bolt"

	"github.com/google/cayley/graph"
	"github.com/google/cayley/graph/iterator"
	"github.com/google/cayley/quad"
)

func init() {
	graph.RegisterTripleStore("bolt", true, newQuadStore, createNewBolt)
}

type Token struct {
	bucket []byte
	key    []byte
}

func (t *Token) Key() interface{} {
	return fmt.Sprint(t.bucket, t.key)
}

type QuadStore struct {
	db         *bolt.DB
	path       string
	open       bool
	size       int64
	horizon    int64
	makeHasher func() hash.Hash
	hasherSize int
}

func createNewBolt(path string, _ graph.Options) error {
	db, err := bolt.Open(path, 0600, nil)
	if err != nil {
		glog.Errorf("Error: couldn't create Bolt database: %v", err)
		return err
	}
	defer db.Close()
	qs := &QuadStore{}
	qs.db = db
	err = qs.createBuckets()
	if err != nil {
		return err
	}
	qs.Close()
	return nil
}

func newQuadStore(path string, options graph.Options) (graph.TripleStore, error) {
	var qs QuadStore
	var err error
	qs.hasherSize = sha1.Size
	qs.makeHasher = sha1.New
	db, err := bolt.Open(path, 0600, nil)
	if err != nil {
		glog.Errorln("Error, couldn't open! ", err)
		return nil, err
	}
	qs.db = db
	err = qs.getMetadata()
	if err != nil {
		return nil, err
	}
	return &qs, nil
}

func (qs *QuadStore) createBuckets() error {
	return qs.db.Update(func(tx *bolt.Tx) error {
		var err error
		for _, index := range [][4]quad.Direction{spo, osp, pos, cps} {
			_, err = tx.CreateBucket(bucketFor(index))
			if err != nil {
				return fmt.Errorf("Couldn't create bucket: %s", err)
			}
		}
		_, err = tx.CreateBucket(logBucket)
		if err != nil {
			return fmt.Errorf("Couldn't create bucket: %s", err)
		}
		_, err = tx.CreateBucket(nodeBucket)
		if err != nil {
			return fmt.Errorf("Couldn't create bucket: %s", err)
		}
		_, err = tx.CreateBucket(metaBucket)
		if err != nil {
			return fmt.Errorf("Couldn't create bucket: %s", err)
		}
		return nil
	})
}

func (qs *QuadStore) Size() int64 {
	return qs.size
}

func (qs *QuadStore) Horizon() int64 {
	return qs.horizon
}

func (qa *QuadStore) createDeltaKeyFor(d *graph.Delta) []byte {
	return []byte(fmt.Sprintf("%018x", d.ID))
}

func bucketFor(d [4]quad.Direction) []byte {
	return []byte{d[0].Prefix(), d[1].Prefix(), d[2].Prefix(), d[3].Prefix()}
}

func (qs *QuadStore) createKeyFor(d [4]quad.Direction, triple quad.Quad) []byte {
	hasher := qs.makeHasher()
	key := make([]byte, 0, (qs.hasherSize * 4))
	key = append(key, qs.convertStringToByteHash(triple.Get(d[0]), hasher)...)
	key = append(key, qs.convertStringToByteHash(triple.Get(d[1]), hasher)...)
	key = append(key, qs.convertStringToByteHash(triple.Get(d[2]), hasher)...)
	key = append(key, qs.convertStringToByteHash(triple.Get(d[3]), hasher)...)
	return key
}

func (qs *QuadStore) createValueKeyFor(s string) []byte {
	hasher := qs.makeHasher()
	key := make([]byte, 0, qs.hasherSize)
	key = append(key, qs.convertStringToByteHash(s, hasher)...)
	return key
}

type IndexEntry struct {
	quad.Quad
	History []int64
}

// Short hand for direction permutations.
var (
	spo       = [4]quad.Direction{quad.Subject, quad.Predicate, quad.Object, quad.Label}
	osp       = [4]quad.Direction{quad.Object, quad.Subject, quad.Predicate, quad.Label}
	pos       = [4]quad.Direction{quad.Predicate, quad.Object, quad.Subject, quad.Label}
	cps       = [4]quad.Direction{quad.Label, quad.Predicate, quad.Subject, quad.Object}
	spoBucket = bucketFor(spo)
	ospBucket = bucketFor(osp)
	posBucket = bucketFor(pos)
	cpsBucket = bucketFor(cps)
)

var logBucket = []byte("log")
var nodeBucket = []byte("node")
var metaBucket = []byte("meta")

func (qs *QuadStore) ApplyDeltas(deltas []*graph.Delta) error {
	var old_size = qs.size
	var old_horizon = qs.horizon
	var size_change int64
	err := qs.db.Update(func(tx *bolt.Tx) error {
		var b *bolt.Bucket
		resizeMap := make(map[string]int64)
		size_change = int64(0)
		for _, d := range deltas {
			bytes, err := json.Marshal(d)
			if err != nil {
				return err
			}
			b = tx.Bucket(logBucket)
			err = b.Put(qs.createDeltaKeyFor(d), bytes)
			if err != nil {
				return err
			}
			err = qs.buildQuadWrite(tx, d.Quad, d.ID, d.Action == graph.Add)
			if err != nil {<commit_msg>slight improvement by appending deltas first<commit_after>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bolt

import (
	"bytes"
	"crypto/sha1"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"hash"

	"github.com/barakmich/glog"
	"github.com/boltdb/bolt"

	"github.com/google/cayley/graph"
	"github.com/google/cayley/graph/iterator"
	"github.com/google/cayley/quad"
)

func init() {
	graph.RegisterTripleStore("bolt", true, newQuadStore, createNewBolt)
}

type Token struct {
	bucket []byte
	key    []byte
}

func (t *Token) Key() interface{} {
	return fmt.Sprint(t.bucket, t.key)
}

type QuadStore struct {
	db         *bolt.DB
	path       string
	open       bool
	size       int64
	horizon    int64
	makeHasher func() hash.Hash
	hasherSize int
}

func createNewBolt(path string, _ graph.Options) error {
	db, err := bolt.Open(path, 0600, nil)
	if err != nil {
		glog.Errorf("Error: couldn't create Bolt database: %v", err)
		return err
	}
	defer db.Close()
	qs := &QuadStore{}
	qs.db = db
	err = qs.createBuckets()
	if err != nil {
		return err
	}
	qs.Close()
	return nil
}

func newQuadStore(path string, options graph.Options) (graph.TripleStore, error) {
	var qs QuadStore
	var err error
	qs.hasherSize = sha1.Size
	qs.makeHasher = sha1.New
	db, err := bolt.Open(path, 0600, nil)
	if err != nil {
		glog.Errorln("Error, couldn't open! ", err)
		return nil, err
	}
	qs.db = db
	err = qs.getMetadata()
	if err != nil {
		return nil, err
	}
	return &qs, nil
}

func (qs *QuadStore) createBuckets() error {
	return qs.db.Update(func(tx *bolt.Tx) error {
		var err error
		for _, index := range [][4]quad.Direction{spo, osp, pos, cps} {
			_, err = tx.CreateBucket(bucketFor(index))
			if err != nil {
				return fmt.Errorf("Couldn't create bucket: %s", err)
			}
		}
		_, err = tx.CreateBucket(logBucket)
		if err != nil {
			return fmt.Errorf("Couldn't create bucket: %s", err)
		}
		_, err = tx.CreateBucket(nodeBucket)
		if err != nil {
			return fmt.Errorf("Couldn't create bucket: %s", err)
		}
		_, err = tx.CreateBucket(metaBucket)
		if err != nil {
			return fmt.Errorf("Couldn't create bucket: %s", err)
		}
		return nil
	})
}

func (qs *QuadStore) Size() int64 {
	return qs.size
}

func (qs *QuadStore) Horizon() int64 {
	return qs.horizon
}

func (qa *QuadStore) createDeltaKeyFor(d *graph.Delta) []byte {
	return []byte(fmt.Sprintf("%018x", d.ID))
}

func bucketFor(d [4]quad.Direction) []byte {
	return []byte{d[0].Prefix(), d[1].Prefix(), d[2].Prefix(), d[3].Prefix()}
}

func (qs *QuadStore) createKeyFor(d [4]quad.Direction, triple quad.Quad) []byte {
	hasher := qs.makeHasher()
	key := make([]byte, 0, (qs.hasherSize * 4))
	key = append(key, qs.convertStringToByteHash(triple.Get(d[0]), hasher)...)
	key = append(key, qs.convertStringToByteHash(triple.Get(d[1]), hasher)...)
	key = append(key, qs.convertStringToByteHash(triple.Get(d[2]), hasher)...)
	key = append(key, qs.convertStringToByteHash(triple.Get(d[3]), hasher)...)
	return key
}

func (qs *QuadStore) createValueKeyFor(s string) []byte {
	hasher := qs.makeHasher()
	key := make([]byte, 0, qs.hasherSize)
	key = append(key, qs.convertStringToByteHash(s, hasher)...)
	return key
}

type IndexEntry struct {
	quad.Quad
	History []int64
}

// Short hand for direction permutations.
var (
	spo       = [4]quad.Direction{quad.Subject, quad.Predicate, quad.Object, quad.Label}
	osp       = [4]quad.Direction{quad.Object, quad.Subject, quad.Predicate, quad.Label}
	pos       = [4]quad.Direction{quad.Predicate, quad.Object, quad.Subject, quad.Label}
	cps       = [4]quad.Direction{quad.Label, quad.Predicate, quad.Subject, quad.Object}
	spoBucket = bucketFor(spo)
	ospBucket = bucketFor(osp)
	posBucket = bucketFor(pos)
	cpsBucket = bucketFor(cps)
)

var logBucket = []byte("log")
var nodeBucket = []byte("node")
var metaBucket = []byte("meta")

func (qs *QuadStore) ApplyDeltas(deltas []*graph.Delta) error {
	var old_size = qs.size
	var old_horizon = qs.horizon
	var size_change int64
	err := qs.db.Update(func(tx *bolt.Tx) error {
		var b *bolt.Bucket
		resizeMap := make(map[string]int64)
		size_change = int64(0)
		for _, d := range deltas {
			bytes, err := json.Marshal(d)
			if err != nil {
				return err
			}
			b = tx.Bucket(logBucket)
			err = b.Put(qs.createDeltaKeyFor(d), bytes)
			if err != nil {
				return err
			}
		}
		for _, d := range deltas {
			err := qs.buildQuadWrite(tx, d.Quad, d.ID, d.Action == graph.Add)
			if err != nil {
{'fraction_non_alphanumeric': 0.14695520885757424, 'fraction_numerical': 0.011474584801207851, 'mean_word_length': 3.6867924528301885, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4476331', 'n_tokens_mistral': 3856, 'n_tokens_neox': 3639, 'n_words': 963}