<commit_before>package commands

import (
	"errors"
	"fmt"
	"io"

	cmds "github.com/jbenet/go-ipfs/commands"
	core "github.com/jbenet/go-ipfs/core"
	internal "github.com/jbenet/go-ipfs/core/commands2/internal"
	importer "github.com/jbenet/go-ipfs/importer"
	"github.com/jbenet/go-ipfs/importer/chunk"
	dag "github.com/jbenet/go-ipfs/merkledag"
	pinning "github.com/jbenet/go-ipfs/pin"
)

// Error indicating the max depth has been exceded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

type AddOutput struct {
	Added []*Object
}

var addCmd = &cmds.Command{
	Options: []cmds.Option{
		cmds.BoolOption("recursive", "r", "Must be specified when adding directories"),
	},
	Arguments: []cmds.Argument{
		cmds.FileArg("file", true, true, "The path to a file to be added to IPFS"),
	},
	Description: "Add an object to ipfs.",
	Help: `Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG. A smarter partial add with a staging area (like git)
remains to be implemented.
`,
	Run: func(req cmds.Request) (interface{}, error) {
		n := req.Context().Node

		readers, err := internal.CastToReaders(req.Arguments())
		if err != nil {
			return nil, err
		}

		dagnodes, err := add(n, readers)
		if err != nil {
			return nil, err
		}

		// TODO: include fs paths in output (will need a way to specify paths in underlying filearg system)
		added := make([]*Object, 0, len(req.Arguments()))
		for _, dagnode := range dagnodes {
			object, err := getOutput(dagnode)
			if err != nil {
				return nil, err
			}

			added = append(added, object)
		}

		return &AddOutput{added}, nil
	},<commit_msg>cmds2/add: temp fix for -r. horrible hack<commit_after>package commands

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	cmds "github.com/jbenet/go-ipfs/commands"
	core "github.com/jbenet/go-ipfs/core"
	internal "github.com/jbenet/go-ipfs/core/commands2/internal"
	importer "github.com/jbenet/go-ipfs/importer"
	"github.com/jbenet/go-ipfs/importer/chunk"
	dag "github.com/jbenet/go-ipfs/merkledag"
	pinning "github.com/jbenet/go-ipfs/pin"
	ft "github.com/jbenet/go-ipfs/unixfs"
)

// Error indicating the max depth has been exceded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

type AddOutput struct {
	Added []*Object
}

var addCmd = &cmds.Command{
	Options: []cmds.Option{
		cmds.BoolOption("recursive", "r", "Must be specified when adding directories"),
	},
	Arguments: []cmds.Argument{
		cmds.StringArg("file", true, true, "The path to a file to be added to IPFS"),
	},
	Description: "Add an object to ipfs.",
	Help: `Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG. A smarter partial add with a staging area (like git)
remains to be implemented.
`,
	Run: func(req cmds.Request) (interface{}, error) {
		n := req.Context().Node

		// THIS IS A HORRIBLE HACK -- FIXME!!!
		// see https://github.com/jbenet/go-ipfs/issues/309
		var added []*Object

		// returns the last one
		addDagnodes := func(dns []*dag.Node) error {
			for _, dn := range dns {
				o, err := getOutput(dn)
				if err != nil {
					return err
				}

				added = append(added, o)
			}
			return nil
		}

		addFile := func(name string) (*dag.Node, error) {
			f, err := os.Open(name)
			if err != nil {
				return nil, err
			}
			defer f.Close()

			dns, err := add(n, []io.Reader{f})
			if err != nil {
				return nil, err
			}

			if err := addDagnodes(dns); err != nil {
				return nil, err
			}
			return dns[len(dns)-1], nil // last dag node is the file.
		}

		var addPath func(name string) (*dag.Node, error)
		addDir := func(name string) (*dag.Node, error) {
			tree := &dag.Node{Data: ft.FolderPBData()}

			entries, err := ioutil.ReadDir(name)
			if err != nil {
				return nil, err
			}

			// construct nodes for containing files.
			for _, e := range entries {
				fp := filepath.Join(name, e.Name())
				nd, err := addPath(fp)
				if err != nil {
					return nil, err
				}

				if err = tree.AddNodeLink(e.Name(), nd); err != nil {
					return nil, err
				}
			}

			log.Infof("adding dir: %s", name)

			if err := addDagnodes([]*dag.Node{tree}); err != nil {
				return nil, err
			}
			return tree, nil
		}

		addPath = func(fpath string) (*dag.Node, error) {
			fi, err := os.Stat(fpath)
			if err != nil {
				return nil, err
			}

			if fi.IsDir() {
				return addDir(fpath)
			}
			return addFile(fpath)
		}

		paths, err := internal.CastToStrings(req.Arguments())
		if err != nil {
			return nil, err
		}

		for _, f := range paths {
			if _, err := addPath(f); err != nil {
				return nil, err
			}
		}

		// readers, err := internal.CastToReaders(req.Arguments())
		// if err != nil {
		// 	return nil, err
		// }
		//
		// dagnodes, err := add(n, readers)
		// if err != nil {
		// 	return nil, err
		// }
		//
		// // TODO: include fs paths in output (will need a way to specify paths in underlying filearg system)
		// added := make([]*Object, 0, len(req.Arguments()))
		// for _, dagnode := range dagnodes {
		// 	object, err := getOutput(dagnode)
		// 	if err != nil {
		// 		return nil, err
		// 	}
		//
		// 	added = append(added, object)
		// }

		return &AddOutput{added}, nil
	},
{'fraction_non_alphanumeric': 0.14734253830149424, 'fraction_numerical': 0.0017022886324947987, 'mean_word_length': 2.9789315274642587, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1510059', 'n_tokens_mistral': 2159, 'n_tokens_neox': 1914, 'n_words': 547}