<commit_before>package goBoom

import (
	"bytes"
	"errors"
	"io"
	"mime/multipart"
	"net/url"
	"path/filepath"
	"strings"
)

type FilesystemService struct {
	c *Client
}

func newFilesystemService(c *Client) *FilesystemService {
	i := &FilesystemService{}
	if c == nil {
		i.c = NewClient(nil)
	} else {
		i.c = c
	}

	return i
}

func (s *FilesystemService) GetULServer() ([]string, error) {
	resp, err := s.c.api.Res("ul/server").Get(nil)
	arr, err := processResponse(resp, err)
	if err != nil {
		return nil, err
	}

	var servers []string
	if err = decodeInto(&servers, arr[1]); err != nil {
		return nil, err
	}

	return servers, nil

}

// Upload pushes the input io.Reader to the service
func (s *FilesystemService) Upload(fname string, input io.Reader) ([]ItemStat, error) {

	servers, err := s.GetULServer()
	if err != nil {
		return nil, err
	}

	if len(servers) < 1 {
		return nil, errors.New("no servers available for upload")
	}

	var bodyBuf bytes.Buffer
	writer := multipart.NewWriter(&bodyBuf)

	part, err := writer.CreateFormFile("file", filepath.Base(fname))
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(part, input)
	if err != nil {
		return nil, err
	}

	err = writer.Close()
	if err != nil {
		return nil, err
	}

	// prepare request
	oldHost := s.c.api.Api.BaseUrl.Host
	s.c.api.Api.BaseUrl.Host = strings.Replace(oldHost, "api.oboom.com", servers[0], 1)

	res := s.c.api.Res("ul")
	res.Payload = &bodyBuf
	res.Headers.Set("Content-Type", writer.FormDataContentType())

	// set  token
	params := map[string]string{
		"token":       s.c.User.session,
		"name_policy": "rename",
		"parent":      "1", // TODO: configure parent
	}

	// do the request
	resp, err := res.FormPost(params)
	arr, err := processResponse(resp, err)
	if err != nil {
		return nil, err
	}
	s.c.api.Api.BaseUrl.Host = oldHost

	var items []ItemStat
	if err = decodeInto(&items, arr[1]); err != nil {
		return nil, err
	}

	return items, nil
}

// Download requests a download url for item
func (s *FilesystemService) Download(item string) (*url.URL, error) {<commit_msg>added RawUpload - copies a MultiPart request from the client directly to the api<commit_after>package goBoom

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"

	"github.com/bitly/go-simplejson"
)

type FilesystemService struct {
	c *Client
}

func newFilesystemService(c *Client) *FilesystemService {
	i := &FilesystemService{}
	if c == nil {
		i.c = NewClient(nil)
	} else {
		i.c = c
	}

	return i
}

func (s *FilesystemService) GetULServer() ([]string, error) {
	resp, err := s.c.api.Res("ul/server").Get(nil)
	arr, err := processResponse(resp, err)
	if err != nil {
		return nil, err
	}

	var servers []string
	if err = decodeInto(&servers, arr[1]); err != nil {
		return nil, err
	}

	return servers, nil

}

// Upload pushes the input io.Reader to the service
func (s *FilesystemService) Upload(fname string, input io.Reader) ([]ItemStat, error) {

	servers, err := s.GetULServer()
	if err != nil {
		return nil, err
	}

	if len(servers) < 1 {
		return nil, errors.New("no servers available for upload")
	}

	var bodyBuf bytes.Buffer
	writer := multipart.NewWriter(&bodyBuf)

	part, err := writer.CreateFormFile("file", filepath.Base(fname))
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(part, input)
	if err != nil {
		return nil, err
	}

	err = writer.Close()
	if err != nil {
		return nil, err
	}

	// prepare request
	oldHost := s.c.api.Api.BaseUrl.Host
	s.c.api.Api.BaseUrl.Host = strings.Replace(oldHost, "api.oboom.com", servers[0], 1)

	res := s.c.api.Res("ul")
	res.Payload = &bodyBuf
	res.Headers.Set("Content-Type", writer.FormDataContentType())

	// set  token
	params := map[string]string{
		"token":       s.c.User.session,
		"name_policy": "rename",
		"parent":      "1", // TODO: configure parent
	}

	// do the request
	resp, err := res.FormPost(params)
	arr, err := processResponse(resp, err)
	if err != nil {
		return nil, err
	}
	s.c.api.Api.BaseUrl.Host = oldHost

	var items []ItemStat
	if err = decodeInto(&items, arr[1]); err != nil {
		return nil, err
	}

	return items, nil
}

// RawUpload expects a multipart io.Reader and pushes it to the service
func (s *FilesystemService) RawUpload(ct string, clen int64, multiBody io.Reader) ([]ItemStat, error) {

	servers, err := s.GetULServer()
	if err != nil {
		return nil, err
	}

	if len(servers) < 1 {
		return nil, errors.New("no servers available for upload")
	}

	// prepare request
	req, err := http.NewRequest("POST", "http://"+servers[0]+"/1.0/ul", multiBody)
	if err != nil {
		return nil, err
	}

	req.ContentLength = clen
	req.Header.Set("Content-Type", ct)

	qry := req.URL.Query()
	qry.Set("token", s.c.User.session)
	qry.Set("name_policy", "rename")
	qry.Set("parent", "1")
	req.URL.RawQuery = qry.Encode()

	resp, err := s.c.c.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	jsonResp, err := simplejson.NewFromReader(resp.Body)
	if err != nil {
		return nil, err
	}

	arr, err := jsonResp.Array()
	if err != nil {
		return nil, err
	}

	if len(arr) < 1 {
		return nil, ErrorResponse{resp, "Illegal oBoom response"}
	}

	statusCode, err := jsonResp.GetIndex(0).Int()
	if err != nil {
		return nil, err
	}

	if statusCode != resp.StatusCode {
		return nil, ErrorResponse{resp, fmt.Sprintf("StatusCode missmatch. %d vs %d\n%+v",
			statusCode,
			resp.StatusCode,
			jsonResp)}
	}

	var items []ItemStat
	if err = decodeInto(&items, arr[1]); err != nil {
		return nil, err
	}

	return items, nil
}

// Download requests a download url for item
func (s *FilesystemService) Download(item string) (*url.URL, error) {
{'fraction_non_alphanumeric': 0.15353428521064694, 'fraction_numerical': 0.0038780186849991186, 'mean_word_length': 3.5139220365950674, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5123178', 'n_tokens_mistral': 2190, 'n_tokens_neox': 2098, 'n_words': 575}