<commit_before>package upload

import (
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	"github.com/materials-commons/gohandy/ezhttp"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/materials-commons/mcstore/server/mcstored/service/rest/upload"
)

// Command contains the arguments and functions for the cli upload command.
var Command = cli.Command{
	Name:    "upload",
	Aliases: []string{"up", "u"},
	Usage:   "Upload data to MaterialsCommons",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "project, p, proj",
			Usage: "Project id to upload to",
		},
		cli.IntFlag{
			Name:  "parallel, n",
			Value: 5,
			Usage: "Number of simultaneous uploads to perform, defaults to 5",
		},
	},
	Action: Cmd,
}

const oneMeg = 1024 * 1024
const twoMeg = oneMeg * 2
const largeFileSize = oneMeg * 25

var project string
var numThreads int

//var pbPool = &pb.Pool{}

// Cmd implements the cli command upload.
func Cmd(c *cli.Context) {
	fmt.Println("upload: ", c.Args())
	if len(c.Args()) != 1 {
		fmt.Println("You must give the directory to walk")
		os.Exit(1)
	}
	dir := c.Args()[0]
	project = c.String("project")
	numThreads = c.Int("parallel")
	fmt.Println(project)
	fmt.Println(numThreads)

	_, errc := files.PWalk(dir, numThreads, processFiles)
	if err := <-errc; err != nil {
		fmt.Println("Got error: ", err)
	}
}

func processFiles(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
	for entry := range entries {
		select {
		case result <- sendFile(entry):
		case <-done:
			fmt.Println("Received done stopping...")
		}
	}
}

func sendFile(fileEntry files.TreeEntry) string {
	createUploadRequest()
	// buf := make([]byte, twoMeg)
	// f, err := os.Open(fileEntry.Path)
	// if err != nil {
	// 	return ""
	// }
	// fileHash := ""
	// // For small files just transfer the bytes. For
	// // large files compute the file hash.
	// if fileEntry.Finfo.Size() > largeFileSize {
	// 	fileHash, _ = file.HashStr(md5.New(), fileEntry.Path)
	// }
	// for {<commit_msg>Remove println. numThreads doesn't need to be global.<commit_after>package upload

import (
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	"github.com/materials-commons/gohandy/ezhttp"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/materials-commons/mcstore/server/mcstored/service/rest/upload"
)

// Command contains the arguments and functions for the cli upload command.
var Command = cli.Command{
	Name:    "upload",
	Aliases: []string{"up", "u"},
	Usage:   "Upload data to MaterialsCommons",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "project, p, proj",
			Usage: "Project id to upload to",
		},
		cli.IntFlag{
			Name:  "parallel, n",
			Value: 5,
			Usage: "Number of simultaneous uploads to perform, defaults to 5",
		},
	},
	Action: Cmd,
}

const oneMeg = 1024 * 1024
const twoMeg = oneMeg * 2
const largeFileSize = oneMeg * 25

var project string

//var pbPool = &pb.Pool{}

// Cmd implements the cli command upload.
func Cmd(c *cli.Context) {
	fmt.Println("upload: ", c.Args())
	if len(c.Args()) != 1 {
		fmt.Println("You must give the directory to walk")
		os.Exit(1)
	}
	dir := c.Args()[0]
	project = c.String("project")
	numThreads := c.Int("parallel")

	_, errc := files.PWalk(dir, numThreads, processFiles)
	if err := <-errc; err != nil {
		fmt.Println("Got error: ", err)
	}
}

func processFiles(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
	for entry := range entries {
		select {
		case result <- sendFile(entry):
		case <-done:
			fmt.Println("Received done stopping...")
		}
	}
}

func sendFile(fileEntry files.TreeEntry) string {
	createUploadRequest()
	// buf := make([]byte, twoMeg)
	// f, err := os.Open(fileEntry.Path)
	// if err != nil {
	// 	return ""
	// }
	// fileHash := ""
	// // For small files just transfer the bytes. For
	// // large files compute the file hash.
	// if fileEntry.Finfo.Size() > largeFileSize {
	// 	fileHash, _ = file.HashStr(md5.New(), fileEntry.Path)
	// }
	// for {
{'fraction_non_alphanumeric': 0.1595771457337025, 'fraction_numerical': 0.008557764913163856, 'mean_word_length': 3.4106548279689233, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2716370', 'n_tokens_mistral': 1524, 'n_tokens_neox': 1420, 'n_words': 376}