<commit_before>/*
Package json-store reads/writes data to json flat files

Given a top level directory, `Get`, `Put`, and `Del` json encoded flat files of
a given name.
*/
package jsonstore

import (
	"encoding/json"
	"io/ioutil"
	"os"
	// "errors"
)

// default dir/file permission
const PERM = 0755

// var MISS = errors.New("File Not Found")

// holds the name of the directory to which the files are written
type Bucket struct {
	prefix string
}

// NewConnection creates our root data dir
func Open(dir string) (*Bucket, error) {

	if len(dir) == 0 {
		return &Bucket{""}, nil
	}

	if dir[:len(dir)-1] == "/" {
		dir = dir[:len(dir)-2]
	}

	if err := os.MkdirAll(dir, PERM); err != nil {
		return nil, err
	}

	return &Bucket{dir}, nil
}

// Get retrieves the contents of the given file and unmarshals it to the given interface
// os.IsNotExist ought to be used to discover if a `Get` can't find a file to avoid panic
func (b *Bucket) Get(key string, v interface{}) error {
	contents, err := ioutil.ReadFile(b.mkkey(key))
	// if os.IsNotExist(err) {
	// 	return MISS
	// }

	if err != nil {
		return err
	}

	err = json.Unmarshal(contents, v)
	if err != nil {
		return err
	}
	return nil
}

// Put marshals and writes the contents of the given interface to the given file
func (b *Bucket) Put(key string, v interface{}) error {
	contents, err := json.Marshal(v)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(b.mkkey(key), contents, PERM)
	if err != nil {
		return err
	}
	return nil
}

// Del deletes the given file
func (b *Bucket) Del(key string) error {
	err := os.Remove(b.mkkey(key))
	if err != nil {
		return err
	}
	return nil
}

// DellAll deletes the top level data dir
func (b *Bucket) DelAll() error {
	err := os.RemoveAll(b.prefix)
	if err != nil {
		return err
	}
	return nil
}

// mkkey creates the full path to the file given the prefix and the key
func (b *Bucket) mkkey(key string) string {
	if len(b.prefix) > 0 {
		b.prefix += "/"
	}
	return b.prefix + key
}
<commit_msg>cleans up docs<commit_after>/*
Package json-store reads/writes data to json flat files

Given a top level directory, `Get`, `Put`, and `Del` json encoded flat files of
a given name.
*/
package jsonstore

import (
	"encoding/json"
	"io/ioutil"
	"os"
	// "errors"
)

// default dir/file permission
const PERM = 0755

// var MISS = errors.New("File Not Found")

// Bucket holds the name of the directory to which the files are written and
// has all the attached methods
type Bucket struct {
	prefix string
}

// NewConnection creates our root data dir
func Open(dir string) (*Bucket, error) {

	if len(dir) == 0 {
		return &Bucket{""}, nil
	}

	if dir[:len(dir)-1] == "/" {
		dir = dir[:len(dir)-2]
	}

	if err := os.MkdirAll(dir, PERM); err != nil {
		return nil, err
	}

	return &Bucket{dir}, nil
}

// Get retrieves the contents of the given file and unmarshals it to the given interface.
// `os.IsNotExist` ought to be used to discover if a `Get` couldn't find a file
func (b *Bucket) Get(key string, v interface{}) error {
	contents, err := ioutil.ReadFile(b.mkkey(key))
	// if os.IsNotExist(err) {
	// 	return MISS
	// }

	if err != nil {
		return err
	}

	err = json.Unmarshal(contents, v)
	if err != nil {
		return err
	}
	return nil
}

// Put marshals and writes the contents of the given interface to the given file
func (b *Bucket) Put(key string, v interface{}) error {
	contents, err := json.Marshal(v)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(b.mkkey(key), contents, PERM)
	if err != nil {
		return err
	}
	return nil
}

// Del deletes the given file
func (b *Bucket) Del(key string) error {
	err := os.Remove(b.mkkey(key))
	if err != nil {
		return err
	}
	return nil
}

// DellAll deletes the top level data dir
func (b *Bucket) DelAll() error {
	err := os.RemoveAll(b.prefix)
	if err != nil {
		return err
	}
	return nil
}

// mkkey creates the full path to the file given the prefix and the key
func (b *Bucket) mkkey(key string) string {
	if len(b.prefix) > 0 {
		b.prefix += "/"
	}
	return b.prefix + key
}

{'fraction_non_alphanumeric': 0.12768584835761917, 'fraction_numerical': 0.00395159298592245, 'mean_word_length': 3.1882109617373318, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3758314', 'n_tokens_mistral': 1483, 'n_tokens_neox': 1423, 'n_words': 536}