<commit_before>package transform

import (
	"errors"
	"io/ioutil"
	"net/http"
	"path/filepath"
)
import (
	pb "code.google.com/p/goprotobuf/proto"
	"manhattan/project"
	tp "tritium/proto"
)

func NewSlugFromFile(filename string) (slug *tp.Slug, err error) {
	var data []byte

	data, err = ioutil.ReadFile(filename)
	if err != nil {
		return
	}

	slug = &tp.Slug{}
	err = pb.Unmarshal(data, slug)
	if err != nil {
		return
	}

	return
}

func NewSlugFromURL(url string) (slug *tp.Slug, err error) {
	var data []byte
	var resp *http.Response

	resp, err = http.Get(url)
	if err != nil {
		return
	}

	if resp.StatusCode != 200 {
		err = errors.New("Couldn't load slug from " + url)
		return
	}

	data, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	if len(data) == 0 {
		err = errors.New("Received blank slug")
		return
	}

	slug = &tp.Slug{}
	err = pb.Unmarshal(data, slug)
	if err != nil {
		return
	}

	return
}

var defaultRewriters = [...]string{
	"requests.ts",
	"response_pre.ts",
	"body.ts",
	"response_post.ts"}

func NewSlugFromProject(proj *project.Project, mixerPath string) (*tp.Slug, error) {
	mxr := tp.OpenMixer(mixerPath)
	rewriters, err := Generate(proj, mxr)
	if err != nil {
		return nil, err
	}

	slug, err := tp.NewSlug(proj.Name, proj.Version, len(defaultRewriters))
	if err != nil {
		return nil, err
	}

	userPackagePath := filepath.Join(proj.Path, "functions", "main.ts")
	LoadFunctions(userPackagePath, mxr.Package)

	for i, name := range defaultRewriters {
		filename := filepath.Join(proj.ScriptPath, name)
		if rewriters[name] == nil {
			return nil, errors.New("Couldn't load rewriter: " + name)
		}
		slug.Transformers[i], err = CompileString(string(rewriters[name]), filename, mxr.Package)
		if err != nil {
			return nil, errors.New("Failed to create " + name + " transformer: " + err.Error())
		}
	}
	return slug, nil
}
<commit_msg>Removing slug from project, that's being moved back to manhattan.<commit_after>package transform

import (
	"errors"
	"io/ioutil"
	"net/http"
	"path/filepath"
)
import (
	pb "code.google.com/p/goprotobuf/proto"
	"manhattan/project"
	tp "tritium/proto"
)

func NewSlugFromFile(filename string) (slug *tp.Slug, err error) {
	var data []byte

	data, err = ioutil.ReadFile(filename)
	if err != nil {
		return
	}

	slug = &tp.Slug{}
	err = pb.Unmarshal(data, slug)
	if err != nil {
		return
	}

	return
}

func NewSlugFromURL(url string) (slug *tp.Slug, err error) {
	var data []byte
	var resp *http.Response

	resp, err = http.Get(url)
	if err != nil {
		return
	}

	if resp.StatusCode != 200 {
		err = errors.New("Couldn't load slug from " + url)
		return
	}

	data, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	if len(data) == 0 {
		err = errors.New("Received blank slug")
		return
	}

	slug = &tp.Slug{}
	err = pb.Unmarshal(data, slug)
	if err != nil {
		return
	}

	return
}

{'fraction_non_alphanumeric': 0.14658774373259054, 'fraction_numerical': 0.002785515320334262, 'mean_word_length': 3.553090332805071, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2801964', 'n_tokens_mistral': 1198, 'n_tokens_neox': 1131, 'n_words': 278}