<commit_before>// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

// Command bzzup uploads files to the swarm HTTP API.
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

func main() {
	var (
		bzzapiFlag    = flag.String("bzzapi", "http://127.0.0.1:8500", "Swarm HTTP endpoint")
		recursiveFlag = flag.Bool("recursive", false, "Upload directories recursively")
		manifestFlag  = flag.Bool("manifest", true, "Skip automatic manifest upload")
	)
	log.SetOutput(os.Stderr)
	log.SetFlags(0)
	flag.Parse()
	if flag.NArg() != 1 {
		log.Fatal("need filename as the first and only argument")
	}

	var (
		file   = flag.Arg(0)
		client = &client{api: *bzzapiFlag}
		mroot  manifest
	)
	fi, err := os.Stat(file)
	if err != nil {
		log.Fatal(err)
	}
	if fi.IsDir() {
		if !*recursiveFlag {
			log.Fatal("argument is a directory and recursive upload is disabled")
		}
		mroot, err = client.uploadDirectory(file)
	} else {
		mroot, err = client.uploadFile(file, fi)
		if *manifestFlag {
			// Wrap the raw file entry in a proper manifest so both hashes get printed.
			mroot = manifest{Entries: []manifest{mroot}}
		}
	}
	if err != nil {
		log.Fatalln("upload failed:", err)
	}
	if *manifestFlag {
		hash, err := client.uploadManifest(mroot)
		if err != nil {
			log.Fatalln("manifest upload failed:", err)
		}
		mroot.Hash = hash
	}

	// Print the manifest. This is the only output to stdout.
	mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
	fmt.Println(string(mrootJSON))
}

// client wraps interaction with the swarm HTTP gateway.
type client struct {
	api string
}

// manifest is the JSON representation of a swarm manifest.
type manifest struct {
	Hash        string     `json:"hash,omitempty"`
	ContentType string     `json:"contentType,omitempty"`
	Path        string     `json:"path,omitempty"`
	Entries     []manifest `json:"entries,omitempty"`
}

func (c *client) uploadFile(file string, fi os.FileInfo) (manifest, error) {
	hash, err := c.uploadFileContent(file, fi)
	m := manifest{
		Hash:        hash,
		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
	}
	return m, err
}

func (c *client) uploadDirectory(dir string) (manifest, error) {
	dirm := manifest{}
	prefix := filepath.ToSlash(dir) + "/"
	err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
		if err != nil || fi.IsDir() {
			return err
		}
		if !strings.HasPrefix(path, dir) {
			return fmt.Errorf("path %s outside directory %s", path, dir)
		}
		entry, err := c.uploadFile(path, fi)
		entry.Path = strings.TrimPrefix(filepath.ToSlash(path), prefix)
		dirm.Entries = append(dirm.Entries, entry)
		return err
	})
	return dirm, err
}

func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
	fd, err := os.Open(file)
	if err != nil {<commit_msg>cmd/bzzup: trim directory in the manifest entry path (#3299)<commit_after>// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

// Command bzzup uploads files to the swarm HTTP API.
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

func main() {
	var (
		bzzapiFlag    = flag.String("bzzapi", "http://127.0.0.1:8500", "Swarm HTTP endpoint")
		recursiveFlag = flag.Bool("recursive", false, "Upload directories recursively")
		manifestFlag  = flag.Bool("manifest", true, "Skip automatic manifest upload")
	)
	log.SetOutput(os.Stderr)
	log.SetFlags(0)
	flag.Parse()
	if flag.NArg() != 1 {
		log.Fatal("need filename as the first and only argument")
	}

	var (
		file   = flag.Arg(0)
		client = &client{api: *bzzapiFlag}
		mroot  manifest
	)
	fi, err := os.Stat(file)
	if err != nil {
		log.Fatal(err)
	}
	if fi.IsDir() {
		if !*recursiveFlag {
			log.Fatal("argument is a directory and recursive upload is disabled")
		}
		mroot, err = client.uploadDirectory(file)
	} else {
		mroot, err = client.uploadFile(file, fi)
		if *manifestFlag {
			// Wrap the raw file entry in a proper manifest so both hashes get printed.
			mroot = manifest{Entries: []manifest{mroot}}
		}
	}
	if err != nil {
		log.Fatalln("upload failed:", err)
	}
	if *manifestFlag {
		hash, err := client.uploadManifest(mroot)
		if err != nil {
			log.Fatalln("manifest upload failed:", err)
		}
		mroot.Hash = hash
	}

	// Print the manifest. This is the only output to stdout.
	mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
	fmt.Println(string(mrootJSON))
}

// client wraps interaction with the swarm HTTP gateway.
type client struct {
	api string
}

// manifest is the JSON representation of a swarm manifest.
type manifest struct {
	Hash        string     `json:"hash,omitempty"`
	ContentType string     `json:"contentType,omitempty"`
	Path        string     `json:"path,omitempty"`
	Entries     []manifest `json:"entries,omitempty"`
}

func (c *client) uploadFile(file string, fi os.FileInfo) (manifest, error) {
	hash, err := c.uploadFileContent(file, fi)
	m := manifest{
		Hash:        hash,
		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
	}
	return m, err
}

func (c *client) uploadDirectory(dir string) (manifest, error) {
	dirm := manifest{}
	prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
	err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
		if err != nil || fi.IsDir() {
			return err
		}
		if !strings.HasPrefix(path, dir) {
			return fmt.Errorf("path %s outside directory %s", path, dir)
		}
		entry, err := c.uploadFile(path, fi)
		entry.Path = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
		dirm.Entries = append(dirm.Entries, entry)
		return err
	})
	return dirm, err
}

func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
	fd, err := os.Open(file)
	if err != nil {
{'fraction_non_alphanumeric': 0.12962187805218361, 'fraction_numerical': 0.0055811357611273895, 'mean_word_length': 3.600770218228498, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1307580', 'n_tokens_mistral': 2538, 'n_tokens_neox': 2379, 'n_words': 772}