<commit_before>package file

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/slinga/object"
	"github.com/Aptomi/aptomi/pkg/slinga/object/store"
	"github.com/mattn/go-zglob"
	"io/ioutil"
	"path/filepath"
	"sort"
	"strings"
)

type FileStore struct {
	store.BaseStore

	path string
}

func (store *FileStore) Open(connection string) error {
	store.path = connection

	return nil
}

func (store *FileStore) loadObjects() (map[object.Key]object.BaseObject, error) {
	files, _ := zglob.Glob(filepath.Join(store.path, "**", "*.yaml"))
	sort.Strings(files)

	result := make(map[object.Key]object.BaseObject, 0)
	for _, f := range files {
		if !strings.Contains(f, "external") {
			objects, err := store.loadObjectsFromFile(f)
			if err != nil {
				return nil, fmt.Errorf("Error while loading objects from file %store: %store", f, err)
			}

			for _, obj := range objects {
				result[obj.GetKey()] = obj
			}
		}
	}

	return result, nil
}

func (store *FileStore) loadObjectsFromFile(path string) ([]object.BaseObject, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("Error while reading file %store: %store", path, err)
	}
	objects, err := store.Codec.UnmarshalMany(data)
	if err != nil {
		return nil, fmt.Errorf("Error while unmarshaling file %store: %store", path, err)
	}
<commit_msg>Fix codec usage in file store prototype<commit_after>package file

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/slinga/object"
	"github.com/Aptomi/aptomi/pkg/slinga/object/store"
	"github.com/mattn/go-zglob"
	"io/ioutil"
	"path/filepath"
	"sort"
	"strings"
)

type FileStore struct {
	store.BaseStore

	path string
}

func (store *FileStore) Open(connection string) error {
	store.path = connection

	return nil
}

func (store *FileStore) loadObjects() (map[object.Key]object.BaseObject, error) {
	files, _ := zglob.Glob(filepath.Join(store.path, "**", "*.yaml"))
	sort.Strings(files)

	result := make(map[object.Key]object.BaseObject, 0)
	for _, f := range files {
		if !strings.Contains(f, "external") {
			objects, err := store.loadObjectsFromFile(f)
			if err != nil {
				return nil, fmt.Errorf("Error while loading objects from file %store: %store", f, err)
			}

			for _, obj := range objects {
				result[obj.GetKey()] = obj
			}
		}
	}

	return result, nil
}

func (store *FileStore) loadObjectsFromFile(path string) ([]object.BaseObject, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("Error while reading file %store: %store", path, err)
	}
	objects, err := store.Codec.UnmarshalOneOrMany(data)
	if err != nil {
		return nil, fmt.Errorf("Error while unmarshaling file %store: %store", path, err)
	}

{'fraction_non_alphanumeric': 0.14953271028037382, 'fraction_numerical': 0.0007476635514018691, 'mean_word_length': 4.176015473887814, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2475088', 'n_tokens_mistral': 1018, 'n_tokens_neox': 964, 'n_words': 232}