<commit_before>package packager

import(
	tp "tritium/proto"
	proto "goprotobuf.googlecode.com/hg/proto"
	api "tritium/api"
	"strings"
	"path/filepath"
	"log4go"
	"os"
)

type Package struct { 
	loaded []*PackageInfo
	location string
	LoadPath string
	FallbackPath string
	OutputFile string
	Log log4go.Logger
	*tp.Package
	Options PackageOptions
}

type PackageInfo struct {
	Name string
	Dependencies []string
	Types []string
}

var DefaultPackagePath = "packages"

func LoadDefaultPackage(path *string) (*Package) {
	if path == nil {
		if os.ShellExpand("$TRITIUM_PACKAGES") != "" {
			DefaultPackagePath = os.ShellExpand("$TRITIUM_PACKAGES")
		}
		path = &DefaultPackagePath
	}

	return buildPackage(*path, nil)
}

func OutputDefaultPackage(path string) (pkg *Package, newFilePath string) {
	pkg = BuildDefaultPackage()
	
	_, err := os.Stat(path)
	
	if err != nil {
		creationErr := os.MkdirAll(path, uint32(0777) ) 
		if creationErr != nil {
			panic("Could not make path(" + path + "). Error:" + creationErr.String())
		}
	}
	
	_, name := filepath.Split(pkg.OutputFile)
	newOutputFile := filepath.Join(path, name)
	
	os.Rename(pkg.OutputFile, newOutputFile)
	
	return pkg, newOutputFile
}

func BuildDefaultPackage() (*Package) {
	options := BuildOptions()
	return buildPackage(DefaultPackagePath, options)
}<commit_msg>I'll leave this to Sean...<commit_after>package packager

import(
	tp "tritium/proto"
	proto "goprotobuf.googlecode.com/hg/proto"
	api "tritium/api"
	"strings"
	"path/filepath"
	"log4go"
	"os"
)

type Package struct { 
	loaded []*PackageInfo
	location string
	LoadPath string
	FallbackPath string
	OutputFile string
	Log log4go.Logger
	*tp.Package
	Options PackageOptions
}

type PackageInfo struct {
	Name string
	Dependencies []string
	Types []string
}

var DefaultPackagePath = "packages"

func LoadDefaultPackage(path *string) (*Package) {
	if path == nil {
		path = &DefaultPackagePath
	}

	return buildPackage(*path, nil)
}

func OutputDefaultPackage(path string) (pkg *Package, newFilePath string) {
	pkg = BuildDefaultPackage()
	
	_, err := os.Stat(path)
	
	if err != nil {
		creationErr := os.MkdirAll(path, uint32(0777) ) 
		if creationErr != nil {
			panic("Could not make path(" + path + "). Error:" + creationErr.String())
		}
	}
	
	_, name := filepath.Split(pkg.OutputFile)
	newOutputFile := filepath.Join(path, name)
	
	os.Rename(pkg.OutputFile, newOutputFile)
	
	return pkg, newOutputFile
}

func BuildDefaultPackage() (*Package) {
	options := BuildOptions()
	return buildPackage(DefaultPackagePath, options)
}
{'fraction_non_alphanumeric': 0.12108150470219435, 'fraction_numerical': 0.006269592476489028, 'mean_word_length': 4.6233480176211454, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2953034', 'n_tokens_mistral': 919, 'n_tokens_neox': 854, 'n_words': 208}