<commit_before>// Package sftp provides a filesystem interface using github.com/pkg/sftp

// +build !plan9

package sftp

import (
	"io"
	"io/ioutil"
	"os"
	"os/user"
	"path"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/pkg/errors"
	"github.com/pkg/sftp"
	"github.com/xanzy/ssh-agent"
	"golang.org/x/crypto/ssh"
	"golang.org/x/net/context"
	"golang.org/x/time/rate"
)

const (
	connectionsPerSecond = 10 // don't make more than this many ssh connections/s
)

var (
	currentUser = readCurrentUser()
)

func init() {
	fsi := &fs.RegInfo{
		Name:        "sftp",
		Description: "SSH/SFTP Connection",
		NewFs:       NewFs,
		Options: []fs.Option{{
			Name:     "host",
			Help:     "SSH host to connect to",
			Optional: false,
			Examples: []fs.OptionExample{{
				Value: "example.com",
				Help:  "Connect to example.com",
			}},
		}, {
			Name:     "user",
			Help:     "SSH username, leave blank for current username, " + currentUser,
			Optional: true,
		}, {
			Name:     "port",
			Help:     "SSH port, leave blank to use default (22)",
			Optional: true,
		}, {
			Name:       "pass",
			Help:       "SSH password, leave blank to use ssh-agent.",
			Optional:   true,
			IsPassword: true,
		}, {
			Name:     "key_file",
			Help:     "Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent.",
			Optional: true,
		}, {
			Name:     "use_insecure_cipher",
			Help:     "Enable the user of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker..",
			Optional: true,
			Examples: []fs.OptionExample{
				{
					Value: "false",
					Help:  "Use default Cipher list.",
				}, {
					Value: "true",
					Help:  "Enables the use of the aes128-cbc cipher.",
				},
			},
		}, {
			Name:     "disable_hashcheck",
			Help:     "Disable the exectution of SSH commands to determine if remote file hashing is available, leave blank unless you know what you are doing.",
			Optional: true,
		}},
	}
	fs.Register(fsi)
}

// Fs stores the interface to the remote SFTP files
type Fs struct {
	name         string
	root         string
	features     *fs.Features // optional features
	config       *ssh.ClientConfig
	host         string
	port         string
	url          string
	mkdirLock    *stringLock
	cachedHashes *fs.HashSet
	poolMu       sync.Mutex
	pool         []*conn
	connLimit    *rate.Limiter // for limiting number of connections per second
}

// Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
type Object struct {
	fs      *Fs
	remote  string
	size    int64       // size of the object
	modTime time.Time   // modification time of the object
	mode    os.FileMode // mode bits from the file
	md5sum  *string     // Cached MD5 checksum
	sha1sum *string     // Cached SHA1 checksum
}

// ObjectReader holds the sftp.File interface to a remote SFTP file opened for reading
type ObjectReader struct {
	object   *Object
	sftpFile *sftp.File
}

// readCurrentUser finds the current user name or "" if not found
func readCurrentUser() (userName string) {
	usr, err := user.Current()
	if err == nil {
		return usr.Username
	}
	// Fall back to reading $HOME then $LOGNAME
	userName = os.Getenv("HOME")
	if userName != "" {
		return userName
	}
	return os.Getenv("LOGNAME")
}<commit_msg>sftp: read $USER in username fallback not $HOME<commit_after>// Package sftp provides a filesystem interface using github.com/pkg/sftp

// +build !plan9

package sftp

import (
	"io"
	"io/ioutil"
	"os"
	"os/user"
	"path"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/pkg/errors"
	"github.com/pkg/sftp"
	"github.com/xanzy/ssh-agent"
	"golang.org/x/crypto/ssh"
	"golang.org/x/net/context"
	"golang.org/x/time/rate"
)

const (
	connectionsPerSecond = 10 // don't make more than this many ssh connections/s
)

var (
	currentUser = readCurrentUser()
)

func init() {
	fsi := &fs.RegInfo{
		Name:        "sftp",
		Description: "SSH/SFTP Connection",
		NewFs:       NewFs,
		Options: []fs.Option{{
			Name:     "host",
			Help:     "SSH host to connect to",
			Optional: false,
			Examples: []fs.OptionExample{{
				Value: "example.com",
				Help:  "Connect to example.com",
			}},
		}, {
			Name:     "user",
			Help:     "SSH username, leave blank for current username, " + currentUser,
			Optional: true,
		}, {
			Name:     "port",
			Help:     "SSH port, leave blank to use default (22)",
			Optional: true,
		}, {
			Name:       "pass",
			Help:       "SSH password, leave blank to use ssh-agent.",
			Optional:   true,
			IsPassword: true,
		}, {
			Name:     "key_file",
			Help:     "Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent.",
			Optional: true,
		}, {
			Name:     "use_insecure_cipher",
			Help:     "Enable the user of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker..",
			Optional: true,
			Examples: []fs.OptionExample{
				{
					Value: "false",
					Help:  "Use default Cipher list.",
				}, {
					Value: "true",
					Help:  "Enables the use of the aes128-cbc cipher.",
				},
			},
		}, {
			Name:     "disable_hashcheck",
			Help:     "Disable the exectution of SSH commands to determine if remote file hashing is available, leave blank unless you know what you are doing.",
			Optional: true,
		}},
	}
	fs.Register(fsi)
}

// Fs stores the interface to the remote SFTP files
type Fs struct {
	name         string
	root         string
	features     *fs.Features // optional features
	config       *ssh.ClientConfig
	host         string
	port         string
	url          string
	mkdirLock    *stringLock
	cachedHashes *fs.HashSet
	poolMu       sync.Mutex
	pool         []*conn
	connLimit    *rate.Limiter // for limiting number of connections per second
}

// Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
type Object struct {
	fs      *Fs
	remote  string
	size    int64       // size of the object
	modTime time.Time   // modification time of the object
	mode    os.FileMode // mode bits from the file
	md5sum  *string     // Cached MD5 checksum
	sha1sum *string     // Cached SHA1 checksum
}

// ObjectReader holds the sftp.File interface to a remote SFTP file opened for reading
type ObjectReader struct {
	object   *Object
	sftpFile *sftp.File
}

// readCurrentUser finds the current user name or "" if not found
func readCurrentUser() (userName string) {
	usr, err := user.Current()
	if err == nil {
		return usr.Username
	}
	// Fall back to reading $USER then $LOGNAME
	userName = os.Getenv("USER")
	if userName != "" {
		return userName
	}
	return os.Getenv("LOGNAME")
}
{'fraction_non_alphanumeric': 0.11132580261593342, 'fraction_numerical': 0.005053507728894174, 'mean_word_length': 2.677049180327869, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4762029', 'n_tokens_mistral': 2502, 'n_tokens_neox': 2223, 'n_words': 739}