<commit_before>// client
package main

// TODO: for encryption, make sure use Message Authentication Code
// Alternatively, you can apply your own message authentication, as
// follows. First, encrypt the message using an appropriate
// symmetric-key encryption scheme (e.g., AES-CBC). Then, take the
// entire ciphertext (including any IVs, nonces, or other values
// needed for decryption), apply a message authentication code (e.g.,
// AES-CMAC, SHA1-HMAC, SHA256-HMAC), and append the resulting MAC
// digest to the ciphertext before transmission. On the receiving
// side, check that the MAC digest is valid before decrypting. This is
// known as the encrypt-then-authenticate construction. (See also: 1,
// 2.) This also works fine, but requires a little more care from you.

import (
	"bytes"
	"crypto/rc4"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const (
	MAX_DIR_NAMES   = 1000
	RC4_TRASH_BYTES = 256
)

////////////////////////////////////////
// repository root
////////////////////////////////////////

func repositoryRoot() (repos string, err error) {
	pwd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	// start here, work way up until find .amber in pwd
	repos = pwd
	for {
		pathname := fmt.Sprintf("%s%c.amber", repos, filepath.Separator)
		fi, err := os.Stat(pathname)
		if err == nil {
			if fi.IsDir() {
				repos = pathname
				break // found it
			}
			return "", ErrNoRepos
		}
		if !os.IsNotExist(err) {
			return "", err // some other error
		}
		// have not found it yet
		parent := filepath.Dir(repos)
		if parent == repos {
			return "", ErrNoRepos
		}
		repos = parent
	}
	return repos, nil
}

////////////////////////////////////////
// encryption / decryption
////////////////////////////////////////

func selectIV(algorithm, hName string, blob []byte) (iv []byte, err error) {
	var ivSize int
	var hash string

	switch {
	case strings.HasSuffix(algorithm, "128"):
		ivSize = 16
	case strings.HasSuffix(algorithm, "192"):
		ivSize = 24
	case strings.HasSuffix(algorithm, "256"):
		ivSize = 32
	case algorithm == "rc4":
		return
	default:
		return nil, fmt.Errorf("unknown encryption algorithm: " + algorithm)
	}

	size := fmt.Sprintf("%d", len(blob))
	hash, err = computeHash(hName, []byte(size))
	if err != nil {
		return
	}
	iv = make([]byte, ivSize)
	copy(iv, []byte(hash))
	return
}

func encrypt(blob []byte, algorithm, key string, iv []byte) ([]byte, error) {
	switch {
	case algorithm == "-":
		return blob, nil
		// case strings.HasPrefix(algorithm, "aes"):
		//	// FIXME: key must be 16, 24, or 32 bytes for AES-128, -192, or -256
		//	c,err = aes.NewCipher([]byte(key))
		//	if err != nil {
		//		return nil, err
		//	}
		//	eblob = make([]byte, len(blob)) // padding?
		//	// TODO
		//	return eblob, nil
	case strings.HasPrefix(algorithm, "rc4"):
		return encryptRC4(blob, key)
	}
	return nil, fmt.Errorf("unknown encryption algorithm: %s", algorithm)
}

func encryptRC4(blob []byte, key string) ([]byte, error) {
	// TODO: assert key is between 1 and 256 bytes
	rc4Key := make([]byte, 256)
	copy(rc4Key, []byte(key))
	c, err := rc4.NewCipher(rc4Key)
	if err != nil {
		return nil, err
	}
	// throw away at least 256 bytes of data
	junk := make([]byte, RC4_TRASH_BYTES)
	// ??? how does decryption work with below reseeding
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	for i := 0; i < len(junk); i++ {
		junk[i] = byte(r.Intn(256))
	}
	c.XORKeyStream(junk, junk)
	// actual encryption
	c.XORKeyStream(blob, blob)
	c.Reset()
	return blob, nil
}

func decrypt(blob []byte, algorithm, key string, iv []byte) ([]byte, error) {
	switch {
	case algorithm == "-":
		return blob, nil
	case strings.HasPrefix(algorithm, "rc4"):
		return encryptRC4(blob, key)
	}
	return nil, fmt.Errorf("unknown encryption algorithm: " + algorithm)
}

////////////////////////////////////////
// commit
//
// directory loaded into cache (pcache, then encrypted into ecache)
// new tip created
////////////////////////////////////////

func commit(pathname string) (err error) {
	root, err := repositoryRoot()
	if err != nil {
		return
	}
	meta := new(metadata)
	// TODO: should be loaded from config
	meta.hName = DefaultHash
	meta.eName = DefaultEncryption
	meta.uName = "-"
	return commitPathname(root, pathname, meta)
}

func commitPathname(repositoryRoot, pathname string, meta *metadata) (err error) {
	// log.Println("COMMIT PATHNAME:", pathname)
	fi, err := os.Stat(pathname)
	if err != nil {
		return
	}
	mode := fi.Mode()
	meta.Mode = fmt.Sprintf("0%o", mode)
	meta.Name = fi.Name()
	switch {
	case mode&os.ModeDir != 0:
		err = commitDirectory(repositoryRoot, pathname, meta)
	case mode&os.ModeSymlink != 0:
		err = fmt.Errorf("TODO: implement commitSymlink")
	default:
		err = commitFile(repositoryRoot, pathname, meta)
	}
	return
}

func commitDirectory(repositoryRoot, pathname string, meta *metadata) (err error) {
	log.Println("COMMIT DIRECTORY:", pathname)
	meta.Type = "directory"
	fh, err := os.Open(pathname)
	if err != nil {
		return
	}
	defer fh.Close()

	meta.Children = make([]metadata, 0, MAX_DIR_NAMES)
	for {
		var names []string
		names, err = fh.Readdirnames(MAX_DIR_NAMES)
		if err != nil {
			if err == io.EOF {
				break // done
			}
			return
		}
		for _, name := range names {
			switch {
			case name == "." || name == "..":
			case name == ".amber" || name == ".git":
			default:
				childMeta := new(metadata)
				childMeta.hName = meta.hName
				childMeta.eName = meta.eName
				childMeta.uName = meta.uName
				childName := fmt.Sprintf("%s/%s", pathname, name)
				if err = commitPathname(repositoryRoot, childName, childMeta); err != nil {
					return
				}
				meta.Children = append(meta.Children, *childMeta) // ??? how efficient with large directories ???
			}
		}
	}
	blob, err := json.Marshal(meta.Children)
	if err != nil {
		return
	}
	if err = commitBytes(repositoryRoot, blob, meta); err != nil {
		return
	}
	// once directory committed, do not want to propagate Children up
	meta.Children = nil
	return
}

func commitFile(repositoryRoot, pathname string, meta *metadata) (err error) {
	log.Println("COMMIT FILE:", pathname)
	meta.Type = "file"
	plainBytes, err := ioutil.ReadFile(pathname)
	if err != nil {
		return
	}
	if err = commitBytes(repositoryRoot, plainBytes, meta); err != nil {
		return
	}
	return
}

func commitBytes(repositoryRoot string, blob []byte, meta *metadata) (err error) {
	meta.Phash, err = computeHash(meta.hName, blob)
	if err != nil {
		return
	}
	fname := fmt.Sprintf("%s/pcache/resource/%s", repositoryRoot, meta.Phash)
	if _, err = os.Stat(fname); os.IsNotExist(err) {
		if err = writeFile(fname, blob); err != nil {
			return
		}
	}
	iv, err := selectIV(meta.eName, meta.hName, blob)
	if err != nil {
		return
	}
	cipherBytes, err := encrypt(blob, meta.eName, meta.Phash, iv)
	if err != nil {
		return
	}

	meta.Chash, err = computeHash(meta.hName, cipherBytes)
	if err != nil {
		return
	}
	fname = fmt.Sprintf("%s/ecache/resource/%s", repositoryRoot, meta.Chash)
	if _, err = os.Stat(fname); os.IsNotExist(err) {
		if err = writeFile(fname, cipherBytes); err != nil {
			return
		}
	}
	meta.size = fmt.Sprintf("%d", len(blob))
	return
}

////////////////////////////////////////
// push
//
// all resources not on remote is copied to remote
////////////////////////////////////////

////////////////////////////////////////
// pull<commit_msg>simplified expression<commit_after>// client
package main

// TODO: for encryption, make sure use Message Authentication Code
// Alternatively, you can apply your own message authentication, as
// follows. First, encrypt the message using an appropriate
// symmetric-key encryption scheme (e.g., AES-CBC). Then, take the
// entire ciphertext (including any IVs, nonces, or other values
// needed for decryption), apply a message authentication code (e.g.,
// AES-CMAC, SHA1-HMAC, SHA256-HMAC), and append the resulting MAC
// digest to the ciphertext before transmission. On the receiving
// side, check that the MAC digest is valid before decrypting. This is
// known as the encrypt-then-authenticate construction. (See also: 1,
// 2.) This also works fine, but requires a little more care from you.

import (
	"bytes"
	"crypto/rc4"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const (
	MAX_DIR_NAMES   = 1000
	RC4_TRASH_BYTES = 256
)

////////////////////////////////////////
// repository root
////////////////////////////////////////

func repositoryRoot() (repos string, err error) {
	pwd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	// start here, work way up until find .amber in pwd
	repos = pwd
	for {
		pathname := fmt.Sprintf("%s%c.amber", repos, filepath.Separator)
		fi, err := os.Stat(pathname)
		if err == nil {
			if fi.IsDir() {
				repos = pathname
				break // found it
			}
			return "", ErrNoRepos
		}
		if !os.IsNotExist(err) {
			return "", err // some other error
		}
		// have not found it yet
		parent := filepath.Dir(repos)
		if parent == repos {
			return "", ErrNoRepos
		}
		repos = parent
	}
	return repos, nil
}

////////////////////////////////////////
// encryption / decryption
////////////////////////////////////////

func selectIV(algorithm, hName string, blob []byte) (iv []byte, err error) {
	var ivSize int
	var hash string

	switch {
	case strings.HasSuffix(algorithm, "128"):
		ivSize = 16
	case strings.HasSuffix(algorithm, "192"):
		ivSize = 24
	case strings.HasSuffix(algorithm, "256"):
		ivSize = 32
	case algorithm == "rc4":
		return
	default:
		return nil, fmt.Errorf("unknown encryption algorithm: " + algorithm)
	}

	size := fmt.Sprintf("%d", len(blob))
	hash, err = computeHash(hName, []byte(size))
	if err != nil {
		return
	}
	iv = make([]byte, ivSize)
	copy(iv, []byte(hash))
	return
}

func encrypt(blob []byte, algorithm, key string, iv []byte) ([]byte, error) {
	switch {
	case algorithm == "-":
		return blob, nil
		// case strings.HasPrefix(algorithm, "aes"):
		//	// FIXME: key must be 16, 24, or 32 bytes for AES-128, -192, or -256
		//	c,err = aes.NewCipher([]byte(key))
		//	if err != nil {
		//		return nil, err
		//	}
		//	eblob = make([]byte, len(blob)) // padding?
		//	// TODO
		//	return eblob, nil
	case strings.HasPrefix(algorithm, "rc4"):
		return encryptRC4(blob, key)
	}
	return nil, fmt.Errorf("unknown encryption algorithm: %s", algorithm)
}

func encryptRC4(blob []byte, key string) ([]byte, error) {
	// TODO: assert key is between 1 and 256 bytes
	rc4Key := make([]byte, 256)
	copy(rc4Key, []byte(key))
	c, err := rc4.NewCipher(rc4Key)
	if err != nil {
		return nil, err
	}
	// throw away at least 256 bytes of data
	junk := make([]byte, RC4_TRASH_BYTES)
	// ??? how does decryption work with below reseeding
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	for i := 0; i < len(junk); i++ {
		junk[i] = byte(r.Intn(256))
	}
	c.XORKeyStream(junk, junk)
	// actual encryption
	c.XORKeyStream(blob, blob)
	c.Reset()
	return blob, nil
}

func decrypt(blob []byte, algorithm, key string, iv []byte) ([]byte, error) {
	switch {
	case algorithm == "-":
		return blob, nil
	case strings.HasPrefix(algorithm, "rc4"):
		return encryptRC4(blob, key)
	}
	return nil, fmt.Errorf("unknown encryption algorithm: " + algorithm)
}

////////////////////////////////////////
// commit
//
// directory loaded into cache (pcache, then encrypted into ecache)
// new tip created
////////////////////////////////////////

func commit(pathname string) (err error) {
	root, err := repositoryRoot()
	if err != nil {
		return
	}
	meta := new(metadata)
	// TODO: should be loaded from config
	meta.hName = DefaultHash
	meta.eName = DefaultEncryption
	meta.uName = "-"
	return commitPathname(root, pathname, meta)
}

func commitPathname(repositoryRoot, pathname string, meta *metadata) (err error) {
	// log.Println("COMMIT PATHNAME:", pathname)
	fi, err := os.Stat(pathname)
	if err != nil {
		return
	}
	mode := fi.Mode()
	meta.Mode = fmt.Sprintf("0%o", mode)
	meta.Name = fi.Name()
	switch {
	case mode&os.ModeDir != 0:
		err = commitDirectory(repositoryRoot, pathname, meta)
	case mode&os.ModeSymlink != 0:
		err = fmt.Errorf("TODO: implement commitSymlink")
	default:
		err = commitFile(repositoryRoot, pathname, meta)
	}
	return
}

func commitDirectory(repositoryRoot, pathname string, meta *metadata) (err error) {
	log.Println("COMMIT DIRECTORY:", pathname)
	meta.Type = "directory"
	fh, err := os.Open(pathname)
	if err != nil {
		return
	}
	defer fh.Close()

	meta.Children = make([]metadata, 0, MAX_DIR_NAMES)
	for {
		var names []string
		names, err = fh.Readdirnames(MAX_DIR_NAMES)
		if err != nil {
			if err == io.EOF {
				break // done
			}
			return
		}
		for _, name := range names {
			switch {
			case name == "." || name == "..":
			case name == ".amber" || name == ".git":
			default:
				childMeta := new(metadata)
				childMeta.hName = meta.hName
				childMeta.eName = meta.eName
				childMeta.uName = meta.uName
				childName := fmt.Sprintf("%s/%s", pathname, name)
				if err = commitPathname(repositoryRoot, childName, childMeta); err != nil {
					return
				}
				meta.Children = append(meta.Children, *childMeta) // ??? how efficient with large directories ???
			}
		}
	}
	blob, err := json.Marshal(meta.Children)
	if err != nil {
		return
	}
	if err = commitBytes(repositoryRoot, blob, meta); err != nil {
		return
	}
	// once directory committed, do not want to propagate Children up
	meta.Children = nil
	return
}

func commitFile(repositoryRoot, pathname string, meta *metadata) (err error) {
	log.Println("COMMIT FILE:", pathname)
	meta.Type = "file"
	plainBytes, err := ioutil.ReadFile(pathname)
	if err != nil {
		return
	}
	return commitBytes(repositoryRoot, plainBytes, meta)
}

func commitBytes(repositoryRoot string, blob []byte, meta *metadata) (err error) {
	meta.size = fmt.Sprintf("%d", len(blob))
	meta.Phash, err = computeHash(meta.hName, blob)
	if err != nil {
		return
	}
	fname := fmt.Sprintf("%s/pcache/resource/%s", repositoryRoot, meta.Phash)
	if _, err = os.Stat(fname); os.IsNotExist(err) {
		if err = writeFile(fname, blob); err != nil {
			return
		}
	}
	iv, err := selectIV(meta.eName, meta.hName, blob)
	if err != nil {
		return
	}
	cipherBytes, err := encrypt(blob, meta.eName, meta.Phash, iv)
	if err != nil {
		return
	}

	meta.Chash, err = computeHash(meta.hName, cipherBytes)
	if err != nil {
		return
	}
	fname = fmt.Sprintf("%s/ecache/resource/%s", repositoryRoot, meta.Chash)
	if _, err = os.Stat(fname); os.IsNotExist(err) {
		if err = writeFile(fname, cipherBytes); err != nil {
			return
		}
	}
	return
}

////////////////////////////////////////
// push
//
// all resources not on remote is copied to remote
////////////////////////////////////////

////////////////////////////////////////
// pull
{'fraction_non_alphanumeric': 0.18618739903069467, 'fraction_numerical': 0.00996230479267636, 'mean_word_length': 2.8281370780726616, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3561659', 'n_tokens_mistral': 5392, 'n_tokens_neox': 4975, 'n_words': 1465}