<commit_before>package rainforest

import (
	"crypto/md5"
	"encoding/hex"
	"io/ioutil"
	"mime"
	"os"
	"path/filepath"
	"strconv"
)

// UploadedFile represents a file that has been uploaded to Rainforest
type UploadedFile struct {
	ID        int    `json:"id"`
	Signature string `json:"signature"`
	Digest    string `json:"digest"`
	MimeType  string `json:"mime_type"`
	Size      int64  `json:"size"`
	Name      string `json:"name"`
}

// GetUploadedFiles returns information for all all files uploaded to the
// given test before.
func (c *Client) GetUploadedFiles(fileID int) ([]UploadedFile, error) {
	req, err := c.NewRequest("GET", "tests/"+strconv.Itoa(fileID)+"/files", nil)
	if err != nil {
		return nil, err
	}

	var fileResp []UploadedFile
	_, err = c.Do(req, &fileResp)
	return fileResp, err
}

// AWSFileInfo represents the response when uploading new file data to Rainforest.
// It contains information used to upload data the file to AWS.
type AWSFileInfo struct {
	FileID        int    `json:"file_id"`
	FileSignature string `json:"file_signature"`
	AWSURL        string `json:"aws_url"`
	AWSKey        string `json:"aws_key"`
	AWSAccessID   string `json:"aws_access_id"`
	AWSPolicy     string `json:"aws_policy"`
	AWSACL        string `json:"aws_acl"`
	AWSSignature  string `json:"aws_signature"`
}

// CreateFile creates a UploadedFile resource by sending file information to
// Rainforest. This information is used for uploading the actual file to AWS.
func (c *Client) CreateFile(testID int, file *os.File) (*AWSFileInfo, error) {
	awsFileInfo := &AWSFileInfo{}
	fileName := file.Name()
	fileInfo, err := os.Stat(fileName)

	if err != nil {
		return awsFileInfo, err
	}

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return awsFileInfo, err
	}

	md5CheckSum := md5.Sum(data)
	hexDigest := hex.EncodeToString(md5CheckSum[:16])

	body := UploadedFile{
		MimeType: mime.TypeByExtension(filepath.Ext(fileName)),
		Size:     fileInfo.Size(),
		Name:     fileName,
		Digest:   hexDigest,
	}

	url := "tests/" + strconv.Itoa(testID) + "/files"
	req, err := c.NewRequest("POST", url, body)
	if err != nil {
		return awsFileInfo, err
	}

	_, err = c.Do(req, awsFileInfo)
	return awsFileInfo, err
}<commit_msg>implement upload of file contents<commit_after>package rainforest

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strconv"

	"github.com/urfave/cli"
)

// UploadedFile represents a file that has been uploaded to Rainforest
type UploadedFile struct {
	ID        int    `json:"id"`
	Signature string `json:"signature"`
	Digest    string `json:"digest"`
	MimeType  string `json:"mime_type"`
	Size      int64  `json:"size"`
	Name      string `json:"name"`
}

// GetUploadedFiles returns information for all all files uploaded to the
// given test before.
func (c *Client) GetUploadedFiles(fileID int) ([]UploadedFile, error) {
	req, err := c.NewRequest("GET", "tests/"+strconv.Itoa(fileID)+"/files", nil)
	if err != nil {
		return nil, err
	}

	var fileResp []UploadedFile
	_, err = c.Do(req, &fileResp)
	return fileResp, err
}

// AWSFileInfo represents the response when uploading new file data to Rainforest.
// It contains information used to upload data the file to AWS.
type AWSFileInfo struct {
	FileID        int    `json:"file_id"`
	FileSignature string `json:"file_signature"`
	AWSURL        string `json:"aws_url"`
	AWSKey        string `json:"aws_key"`
	AWSAccessID   string `json:"aws_access_id"`
	AWSPolicy     string `json:"aws_policy"`
	AWSACL        string `json:"aws_acl"`
	AWSSignature  string `json:"aws_signature"`
}

// CreateTestFile creates a UploadedFile resource by sending file information to
// Rainforest. This information is used for uploading the actual file to AWS.
func (c *Client) CreateTestFile(testID int, file *os.File) (*AWSFileInfo, error) {
	awsFileInfo := &AWSFileInfo{}
	fileName := file.Name()
	fileInfo, err := os.Stat(fileName)

	if err != nil {
		return awsFileInfo, err
	}

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return awsFileInfo, err
	}

	md5CheckSum := md5.Sum(data)
	hexDigest := hex.EncodeToString(md5CheckSum[:16])

	body := UploadedFile{
		MimeType: mime.TypeByExtension(filepath.Ext(fileName)),
		Size:     fileInfo.Size(),
		Name:     fileName,
		Digest:   hexDigest,
	}

	url := "tests/" + strconv.Itoa(testID) + "/files"
	req, err := c.NewRequest("POST", url, body)
	if err != nil {
		return awsFileInfo, err
	}

	_, err = c.Do(req, awsFileInfo)
	return awsFileInfo, err
}

// UploadTestFile is a function that uploads the actual file contents to AWS
func (c *Client) UploadTestFile(file *os.File, awsFileInfo *AWSFileInfo) error {
	fileName := filepath.Base(file.Name())
	fileExt := filepath.Ext(fileName)

	var w io.ReadWriteCloser

	mp := multipart.NewWriter(w)
	mp.CreateFormFile("file", fileName)

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return err
	}

	w.Write(data)

	mp.WriteField("key", awsFileInfo.AWSKey)
	mp.WriteField("AWSAccessKeyId", awsFileInfo.AWSAccessID)
	mp.WriteField("acl", awsFileInfo.AWSACL)
	mp.WriteField("policy", awsFileInfo.AWSPolicy)
	mp.WriteField("signature", awsFileInfo.AWSSignature)
	mp.WriteField("Content-Type", mime.TypeByExtension(fileExt))

	url := awsFileInfo.AWSURL

	var req *http.Request
	req, err = http.NewRequest("POST", url, w)

	if err != nil {
		return err
	}

	req.Header.Set("Content-Type", "multipart/form-data, boundary="+mp.Boundary())

	var resp *http.Response
	resp, err = http.DefaultClient.Do(req)

	if err != nil {
		return err
	}

	status := resp.StatusCode
	if status >= 300 {
		var body []byte
		body, err = ioutil.ReadAll(resp.Body)

		if err != nil {
			return err
		}

		errMsg := fmt.Sprintf("There was an error uploading your file - %v: %v", file.Name(), string(body))
		return cli.NewExitError(errMsg, 1)
	}

	return nil
}
{'fraction_non_alphanumeric': 0.12886773206392382, 'fraction_numerical': 0.0034002040122407345, 'mean_word_length': 4.006808510638298, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1313832', 'n_tokens_mistral': 2148, 'n_tokens_neox': 2076, 'n_words': 559}