<commit_before>package configuration

import (
	"bufio"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/hheld/ResPP/pkg/errors"
)

// File consists of a file name and its base64 encoded content.
type File struct {
	Name           string `json:"name"`
	EncodedContent string `json:"content"`
}

// Resource holds a list of all files sorted under a prefix string.
type Resource struct {
	Prefix string `json:"prefix"`
	Files  []File `json:"files"`
}

// Configuration contains all information that is stored in the config file.
type Configuration struct {
	Contents []Resource `json:"contents"`
}<commit_msg>The configuration file should not include the encoded contents.<commit_after>package configuration

import (
	"bufio"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/hheld/ResPP/pkg/errors"
)

// File consists of a file name and its base64 encoded content.
type File struct {
	Name           string `json:"name"`
	EncodedContent string `json:"-"`
}

// Resource holds a list of all files sorted under a prefix string.
type Resource struct {
	Prefix string `json:"prefix"`
	Files  []File `json:"files"`
}

// Configuration contains all information that is stored in the config file.
type Configuration struct {
	Contents []Resource `json:"contents"`
}
{'fraction_non_alphanumeric': 0.10975609756097561, 'fraction_numerical': 0.006097560975609756, 'mean_word_length': 4.108949416342412, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6940664', 'n_tokens_mistral': 399, 'n_tokens_neox': 391, 'n_words': 144}