<commit_before>package backend

import (
	"sync"
	"errors"
	"bytes"
	"fmt"
	"crypto/sha1"
	"bufio"

	"github.com/tsileo/datadatabase/encrypt"

	"code.google.com/p/go.crypto/nacl/secretbox"
)

var headerSize = 59

type EncryptBackend struct {
	dest BlobHandler
	// index map the plain text hash to encrypted hash
	index map[string]string

	key *[32]byte

	sync.Mutex
}

func NewEncryptBackend(keyPath string, dest BlobHandler) *EncryptBackend {
	if err := encrypt.LoadKey(keyPath); err != nil {
		panic(err)
	}
	b := &EncryptBackend{dest: dest, index:make(map[string]string), key:&encrypt.Key}
	hashes := make(chan string)
	errs := make(chan error)
	go func() {
		errs <- b.dest.Enumerate(hashes)
	
	}()
	for hash := range hashes {
		scanner := b.scanner(hash)
		plainHash, err := scanHash(scanner)
		if err != nil {
			panic(err)
			//return errors.New(fmt.Sprintf("Error reading plain hash from %v, %v", hash, err))
		}
		b.index[plainHash] = hash
	}
	if err := <-errs; err != nil {
		panic(err)
		//return err
	}
	return b
}

func (b *EncryptBackend) Put(hash string, data []byte) (err error) {
	// #datadb/secretbox\n
	// data hash\n
	// data
	var nonce [24]byte
	//out := make([]byte, len(data) + secretbox.Overhead + 24 + headerSize)
	if err := encrypt.GenerateNonce(&nonce); err != nil {
		return err
	}
	var out bytes.Buffer
	out.WriteString("#datadb/secretbox\n")
	out.WriteString(fmt.Sprintf("%v\n", hash))
	encData := make([]byte, len(data) + secretbox.Overhead)
	secretbox.Seal(encData[0:0], data, &nonce, b.key)	
	out.Write(nonce[:])
	out.Write(encData)
	encSha1 := sha1.New()
	encSha1.Write(out.Bytes())
	encHash := fmt.Sprintf("%x", encSha1.Sum(nil))
	b.dest.Put(encHash ,out.Bytes())
	b.Lock()
	b.index[hash] = encHash
	defer b.Unlock()
	return
}

func (b *EncryptBackend) Exists(hash string) bool {
	b.Lock()
	defer b.Unlock()
	_, exists := b.index[hash]
	return exists
}

func (b *EncryptBackend) scanner(hash string) *bufio.Scanner {
	enc, err := b.dest.Get(hash)
	if err != nil {
		return nil
	}
	buf := bytes.NewBuffer(enc)
	return bufio.NewScanner(buf)
}

func scanHash(scanner *bufio.Scanner) (hash string, err error) {
	if !scanner.Scan() {
		return "", errors.New("No line to read")
	}
	if scanner.Text() != "#datadb/secretbox" {
		return "", errors.New("bad header")
	}
	if !scanner.Scan() {
		return "", errors.New("ref not found")
	}
	return scanner.Text(), nil
}

func (b *EncryptBackend) Get(hash string) (data []byte, err error) {
	ref, _ := b.index[hash]
	enc, err := b.dest.Get(ref)
	if err != nil {
		return data, err
	}
	box := enc[headerSize:]
	var nonce [24]byte
	encData := make([]byte, len(box) - 24)
	copy(nonce[:], box[:24])
	copy(encData[:], box[24:])
	out := make([]byte, len(box) - 24)
	out, success := secretbox.Open(nil, encData, &nonce, b.key)
	if !success {
		return data, fmt.Errorf("failed to decrypt blob %v/%v", hash, ref)
	}
	return out, nil
}

func (b *EncryptBackend) Enumerate(blobs chan<- string) error {<commit_msg>EncryptBackend cleanup<commit_after>package backend

import (
	"sync"
	"errors"
	"bytes"
	"fmt"
	"crypto/sha1"
	"bufio"

	"github.com/tsileo/datadatabase/encrypt"

	"code.google.com/p/go.crypto/nacl/secretbox"
	"code.google.com/p/snappy-go/snappy"
)

var headerSize = 59

type EncryptBackend struct {
	dest BlobHandler
	// index map the plain text hash to encrypted hash
	index map[string]string

	// holds the encryption key
	key *[32]byte

	sync.Mutex
}

// NewEncryptBackend return a backend that encrypt/decrypt blobs on the fly,
// blobs are compressed with snappy before encryption with nacl/secretbox.
// At startup it scan encrypted blobs to discover the plain hash (the hash of the plain-text/unencrypted data).
// Blobs are stored in the following format:
//
// #datadb/secretbox\n
// [plain hash]\n
// [encrypted data]
//
func NewEncryptBackend(keyPath string, dest BlobHandler) *EncryptBackend {
	if err := encrypt.LoadKey(keyPath); err != nil {
		panic(err)
	}
	b := &EncryptBackend{dest: dest, index:make(map[string]string), key:&encrypt.Key}
	// Scan the blobs to discover the plain text blob hashes and build the in-memory index
	hashes := make(chan string)
	errs := make(chan error)
	go func() {
		errs <- b.dest.Enumerate(hashes)
	
	}()
	for hash := range hashes {
		scanner := b.scanner(hash)
		plainHash, err := scanHash(scanner)
		if err != nil {
			panic(err)
			//return errors.New(fmt.Sprintf("Error reading plain hash from %v, %v", hash, err))
		}
		b.index[plainHash] = hash
	}
	if err := <-errs; err != nil {
		panic(err)
		//return err
	}
	return b
}

func (b *EncryptBackend) Put(hash string, rawData []byte) (err error) {
	// #datadb/secretbox\n
	// data hash\n
	// data
	var nonce [24]byte
	//out := make([]byte, len(data) + secretbox.Overhead + 24 + headerSize)
	if err := encrypt.GenerateNonce(&nonce); err != nil {
		return err
	}
	// First we compress the data with snappy
	data, err := snappy.Encode(nil, rawData)
	if err != nil {
		return
	}
	var out bytes.Buffer
	out.WriteString("#datadb/secretbox\n")
	out.WriteString(fmt.Sprintf("%v\n", hash))
	encData := make([]byte, len(data) + secretbox.Overhead)
	secretbox.Seal(encData[0:0], data, &nonce, b.key)	
	out.Write(nonce[:])
	out.Write(encData)
	encSha1 := sha1.New()
	encSha1.Write(out.Bytes())
	encHash := fmt.Sprintf("%x", encSha1.Sum(nil))
	b.dest.Put(encHash ,out.Bytes())
	b.Lock()
	b.index[hash] = encHash
	defer b.Unlock()
	return
}

func (b *EncryptBackend) Exists(hash string) bool {
	b.Lock()
	defer b.Unlock()
	_, exists := b.index[hash]
	return exists
}

func (b *EncryptBackend) scanner(hash string) *bufio.Scanner {
	enc, err := b.dest.Get(hash)
	if err != nil {
		return nil
	}
	buf := bytes.NewBuffer(enc)
	return bufio.NewScanner(buf)
}

func scanHash(scanner *bufio.Scanner) (hash string, err error) {
	if !scanner.Scan() {
		return "", errors.New("No line to read")
	}
	if scanner.Text() != "#datadb/secretbox" {
		return "", errors.New("bad header")
	}
	if !scanner.Scan() {
		return "", errors.New("ref not found")
	}
	return scanner.Text(), nil
}

func (b *EncryptBackend) Get(hash string) (data []byte, err error) {
	ref, _ := b.index[hash]
	enc, err := b.dest.Get(ref)
	if err != nil {
		return data, err
	}
	box := enc[headerSize:]
	var nonce [24]byte
	encData := make([]byte, len(box) - 24)
	copy(nonce[:], box[:24])
	copy(encData[:], box[24:])
	out := make([]byte, len(box) - 24)
	out, success := secretbox.Open(nil, encData, &nonce, b.key)
	if !success {
		return data, fmt.Errorf("failed to decrypt blob %v/%v", hash, ref)
	}
	// Decode snappy data
	data, err = snappy.Decode(nil, out)
	if err != nil {
		return data, fmt.Errorf("failed to decode blob %v/%v", hash, ref)
	}
	return 
}

func (b *EncryptBackend) Enumerate(blobs chan<- string) error {
{'fraction_non_alphanumeric': 0.16318053701231272, 'fraction_numerical': 0.007417297136923305, 'mean_word_length': 3.9391941391941394, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4848224', 'n_tokens_mistral': 2672, 'n_tokens_neox': 2583, 'n_words': 657}