<commit_before>package create

import (
	"crypto/tls"
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	"github.com/materials-commons/mcstore/cmd/pkg/opts"
	"github.com/materials-commons/mcstore/cmd/pkg/project"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/parnurzeal/gorequest"
)

type createCommandArgs struct {
	projectName   string
	projectID     string
	directoryPath string
	n             int
}

var (
	// Command contains the arguments and function for the cli project create command.
	Command = cli.Command{
		Name:    "create",
		Aliases: []string{"cr", "c"},
		Usage:   "Create a new project",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "directory, dir, d",
				Usage: "The base directory for the project",
			},
			cli.StringFlag{
				Name:  "projectid, pid, p",
				Usage: "The projectid on Materials Commons",
			},
			cli.IntFlag{
				Name:  "parallel, n",
				Value: 3,
				Usage: "Number of simultaneous uploads to perform, defaults to 3",
			},
			cli.IntFlag{
				Name:  "parallel, n",
				Value: 3,
				Usage: "Number of simultaneous uploads to perform, defaults to 3",
			},
		},
		Action: createCLI,
	}

	args createCommandArgs
)

// createCLI implements the project create command.
func createCLI(c *cli.Context) {
	if err := validateArgs(c); err != nil {
		fmt.Println("Invalid arguments:", err)
		os.Exit(1)
	}

	proj, err := project.Create(args.directoryPath, args.projectName, args.projectID)
	if err != nil {
		fmt.Println("Unable to create project:", err)
		os.Exit(1)
	}

	fmt.Println("Indexing project...")
	indexProject(args.directoryPath, proj)
	fmt.Println("Done.")
}

// validate will validate the command line arguments. It will print a message
// and exit if there is a bad argument.
func validateArgs(c *cli.Context) error {
	if len(c.Args()) != 1 {
		return fmt.Errorf("You must supply a name for the project.")
	}
	args.projectName = c.Args()[0]

	args.projectID = c.String("projectid")
	if err := validateProject(args.projectID); err != nil {
		return err
	}

	args.directoryPath = c.String("directory")
	if err := validateDirectoryPath(args.directoryPath); err != nil {
		return err
	}

	args.n = opts.GetNumThreads(c)

	return nil
}

// validateProject ensures that the projectID exists and the user has access.
func validateProject(projectID string) error {
	if projectID == "" {
		return fmt.Errorf("You must specify a project id from Materials Commons.")
	}
	return nil
}

// validateDirectoryPath checks that the given directory path exists.
func validateDirectoryPath(path string) error {
	if path == "" {
		return fmt.Errorf("You must specify a local directory path where the project files are located.")
	}

	if _, err := os.Stat(path); err != nil {
		return fmt.Errorf("Invalid directory: %s", path)
	}
	return nil
}

// indexProject walks the directory tree and indexes each of the files found. Indexing
// can be performed in parallel.
func indexProject(path string, proj *project.MCProject) error {
	fn := func(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
		indexEntries(proj, done, entries, result)
	}
	walker := &files.PWalker{
		NumParallel: args.n,
		ProcessFn:   fn,
		ProcessDirs: true,
	}
	walker.PWalk(args.directoryPath)<commit_msg>Remove projectid and duplicate parallel flags. Create doesn't have a project id as it is creating the project.<commit_after>package create

import (
	"crypto/tls"
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	"github.com/materials-commons/mcstore/cmd/pkg/opts"
	"github.com/materials-commons/mcstore/cmd/pkg/project"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/parnurzeal/gorequest"
)

type createCommandArgs struct {
	projectName   string
	projectID     string
	directoryPath string
	n             int
}

var (
	// Command contains the arguments and function for the cli project create command.
	Command = cli.Command{
		Name:    "create",
		Aliases: []string{"cr", "c"},
		Usage:   "Create a new project",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "directory, dir, d",
				Usage: "The base directory for the project",
			},
			cli.IntFlag{
				Name:  "parallel, n",
				Value: 3,
				Usage: "Number of simultaneous uploads to perform, defaults to 3",
			},
		},
		Action: createCLI,
	}

	args createCommandArgs
)

// createCLI implements the project create command.
func createCLI(c *cli.Context) {
	if err := validateArgs(c); err != nil {
		fmt.Println("Invalid arguments:", err)
		os.Exit(1)
	}

	proj, err := project.Create(args.directoryPath, args.projectName, args.projectID)
	if err != nil {
		fmt.Println("Unable to create project:", err)
		os.Exit(1)
	}

	fmt.Println("Indexing project...")
	indexProject(args.directoryPath, proj)
	fmt.Println("Done.")
}

// validate will validate the command line arguments. It will print a message
// and exit if there is a bad argument.
func validateArgs(c *cli.Context) error {
	if len(c.Args()) != 1 {
		return fmt.Errorf("You must supply a name for the project.")
	}
	args.projectName = c.Args()[0]

	if err := createProject(args.projectName); err != nil {
		return err
	}

	args.directoryPath = c.String("directory")
	if err := validateDirectoryPath(args.directoryPath); err != nil {
		return err
	}

	args.n = opts.GetNumThreads(c)

	return nil
}

// validateProject ensures that the projectID exists and the user has access.
func createProject(projectName string) error {

	return nil
}

// validateDirectoryPath checks that the given directory path exists.
func validateDirectoryPath(path string) error {
	if path == "" {
		return fmt.Errorf("You must specify a local directory path where the project files are located.")
	}

	if _, err := os.Stat(path); err != nil {
		return fmt.Errorf("Invalid directory: %s", path)
	}
	return nil
}

// indexProject walks the directory tree and indexes each of the files found. Indexing
// can be performed in parallel.
func indexProject(path string, proj *project.MCProject) error {
	fn := func(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
		indexEntries(proj, done, entries, result)
	}
	walker := &files.PWalker{
		NumParallel: args.n,
		ProcessFn:   fn,
		ProcessDirs: true,
	}
	walker.PWalk(args.directoryPath)
{'fraction_non_alphanumeric': 0.1210365362948656, 'fraction_numerical': 0.0022533397714469662, 'mean_word_length': 3.7435114503816793, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5497231', 'n_tokens_mistral': 2130, 'n_tokens_neox': 1963, 'n_words': 617}