<commit_before>// Package ftp interfaces with FTP servers
package ftp

// FIXME Mover and DirMover are possible using c.Rename
// FIXME support conntimeout

import (
	"io"
	"net/textproto"
	"net/url"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/jlaffaye/ftp"
	"github.com/ncw/rclone/fs"
	"github.com/pkg/errors"
)

// Register with Fs
func init() {
	fs.Register(&fs.RegInfo{
		Name:        "ftp",
		Description: "FTP interface",
		NewFs:       NewFs,
		Options: []fs.Option{
			{
				Name: "username",
				Help: "Username",
			}, {
				Name:       "password",
				Help:       "Password",
				IsPassword: true,
			}, {
				Name: "url",
				Help: "FTP URL",
			},
		},
	})
}

// Fs represents a remote FTP server
type Fs struct {
	name     string       // name of this remote
	root     string       // the path we are working on if any
	features *fs.Features // optional features
	url      *url.URL
	user     string
	pass     string
	dialAddr string
	poolMu   sync.Mutex
	pool     []*ftp.ServerConn
}

// Object describes an FTP file
type Object struct {
	fs     *Fs
	remote string
	info   *FileInfo
}

// FileInfo is the metadata known about an FTP file
type FileInfo struct {
	Name    string
	Size    uint64
	ModTime time.Time
	IsDir   bool
}

// ------------------------------------------------------------

// Name of this fs
func (f *Fs) Name() string {
	return f.name
}

// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
	return f.root
}

// String returns a description of the FS
func (f *Fs) String() string {
	return f.url.String()
}

// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
	return f.features
}

// Open a new connection to the FTP server.
func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
	fs.Debugf(f, "Connecting to FTP server")
	c, err := ftp.DialTimeout(f.dialAddr, 30*time.Second)
	if err != nil {
		fs.Errorf(f, "Error while Dialing %s: %s", f.dialAddr, err)
		return nil, errors.Wrap(err, "ftpConnection Dial")
	}
	err = c.Login(f.user, f.pass)
	if err != nil {
		_ = c.Quit()
		fs.Errorf(f, "Error while Logging in into %s: %s", f.dialAddr, err)
		return nil, errors.Wrap(err, "ftpConnection Login")
	}
	return c, nil
}

// Get an FTP connection from the pool, or open a new one
func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
	f.poolMu.Lock()
	if len(f.pool) > 0 {
		c = f.pool[0]
		f.pool = f.pool[1:]
	}
	f.poolMu.Unlock()
	if c != nil {
		return c, nil
	}
	return f.ftpConnection()
}

// Return an FTP connection to the pool
//
// It nils the pointed to connection out so it can't be reused
func (f *Fs) putFtpConnection(c **ftp.ServerConn) {
	f.poolMu.Lock()
	f.pool = append(f.pool, *c)
	f.poolMu.Unlock()
}

// NewFs contstructs an Fs from the path, container:path
func NewFs(name, root string) (ff fs.Fs, err error) {
	// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
	URL := fs.ConfigFileGet(name, "url")
	user := fs.ConfigFileGet(name, "username")
	pass := fs.ConfigFileGet(name, "password")
	pass, err = fs.Reveal(pass)
	if err != nil {
		return nil, errors.Wrap(err, "NewFS decrypt password")
	}
	u, err := url.Parse(URL)
	if err != nil {
		return nil, errors.Wrap(err, "NewFS URL parse")
	}
	urlPath := strings.Trim(u.Path, "/")
	fullPath := root
	if urlPath != "" && !strings.HasPrefix("/", root) {
		fullPath = path.Join(u.Path, root)
	}
	root = fullPath
	dialAddr := u.Host
	if strings.IndexRune(dialAddr, ':') < 0 {
		dialAddr += ":21"
	}
	f := &Fs{
		name:     name,
		root:     root,
		url:      u,
		user:     user,
		pass:     <PASSWORD>,
		dialAddr: dialAddr,
	}
	f.features = (&fs.Features{}).Fill(f)
	// Make a connection and pool it to return errors early
	c, err := f.getFtpConnection()
	if err != nil {
		return nil, errors.Wrap(err, "NewFs")
	}
	f.putFtpConnection(&c)
	if root != "" {
		// Check to see if the root actually an existing file
		remote := path.Base(root)
		f.root = path.Dir(root)
		if f.root == "." {
			f.root = ""
		}
		_, err := f.NewObject(remote)
		if err != nil {
			if err == fs.ErrorObjectNotFound || errors.Cause(err) == fs.ErrorNotAFile {
				// File doesn't exist so return old f
				f.root = root
				return f, nil
			}
			return nil, err
		}
		// return an error with an fs which points to the parent
		return f, fs.ErrorIsFile
	}
	return f, err
}

// translateErrorFile turns FTP errors into rclone errors if possible for a file
func translateErrorFile(err error) error {
	switch errX := err.(type) {
	case *textproto.Error:
		switch errX.Code {
		case ftp.StatusFileUnavailable:
			err = fs.ErrorObjectNotFound
		}
	}
	return err
}

// translateErrorDir turns FTP errors into rclone errors if possible for a directory
func translateErrorDir(err error) error {
	switch errX := err.(type) {
	case *textproto.Error:
		switch errX.Code {
		case ftp.StatusFileUnavailable:
			err = fs.ErrorDirNotFound
		}
	}
	return err
}

// NewObject finds the Object at remote.  If it can't be found
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(remote string) (o fs.Object, err error) {
	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
	fullPath := path.Join(f.root, remote)
	dir := path.Dir(fullPath)
	base := path.Base(fullPath)

	c, err := f.getFtpConnection()
	if err != nil {
		return nil, errors.Wrap(err, "NewObject")
	}
	files, err := c.List(dir)
	f.putFtpConnection(&c)
	if err != nil {
		return nil, translateErrorFile(err)
	}
	for i, file := range files {
		if file.Type != ftp.EntryTypeFolder && file.Name == base {
			o := &Object{
				fs:     f,
				remote: remote,
			}
			info := &FileInfo{
				Name:    remote,
				Size:    files[i].Size,
				ModTime: files[i].Time,
			}
			o.info = info

			return o, nil
		}
	}
	return nil, fs.ErrorObjectNotFound
}

func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) {
	// defer fs.Trace(dir, "curlevel=%d", curlevel)("")
	c, err := f.getFtpConnection()
	if err != nil {
		out.SetError(errors.Wrap(err, "list"))
		return
	}
	files, err := c.List(path.Join(f.root, dir))
	f.putFtpConnection(&c)
	if err != nil {
		out.SetError(translateErrorDir(err))
		return
	}
	for i := range files {
		object := files[i]
		newremote := path.Join(dir, object.Name)
		switch object.Type {
		case ftp.EntryTypeFolder:
			if object.Name == "." || object.Name == ".." {
				continue
			}
			if out.IncludeDirectory(newremote) {
				d := &fs.Dir{
					Name:  newremote,
					When:  object.Time,
					Bytes: 0,
					Count: -1,
				}
				if curlevel < out.Level() {
					f.list(out, path.Join(dir, object.Name), curlevel+1)
				}
				if out.AddDir(d) {
					return
				}
			}
		default:
			o := &Object{
				fs:     f,
				remote: newremote,
			}
			info := &FileInfo{
				Name:    newremote,
				Size:    object.Size,
				ModTime: object.Time,
			}
			o.info = info
			if out.Add(o) {
				return
			}
		}
	}
}

// List the objects and directories of the Fs starting from dir
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound (using out.SetError())
// if the directory isn't found.
//
// Fses must support recursion levels of fs.MaxLevel and 1.
// They may return ErrorLevelNotSupported otherwise.
func (f *Fs) List(out fs.ListOpts, dir string) {
	// defer fs.Trace(dir, "")("")
	f.list(out, dir, 1)
	out.Finished()
}

// Hashes are not supported
func (f *Fs) Hashes() fs.HashSet {
	return 0
}

// Precision shows Modified Time not supported
func (f *Fs) Precision() time.Duration {
	return fs.ModTimeNotSupported
}

// Put in to the remote path with the modTime given of the given size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
	// fs.Debugf(f, "Trying to put file %s", src.Remote())
	o := &Object{
		fs:     f,
		remote: src.Remote(),
	}
	err := o.Update(in, src)
	return o, err
}

// getInfo reads the FileInfo for a path
func (f *Fs) getInfo(remote string) (fi *FileInfo, err error) {
	// defer fs.Trace(remote, "")("fi=%v, err=%v", &fi, &err)
	dir := path.Dir(remote)
	base := path.Base(remote)

	c, err := f.getFtpConnection()
	if err != nil {
		return nil, errors.Wrap(err, "getInfo")
	}
	files, err := c.List(dir)
	f.putFtpConnection(&c)
	if err != nil {
		return nil, translateErrorFile(err)
	}

	for i := range files {
		if files[i].Name == base {
			info := &FileInfo{
				Name:    remote,
				Size:    files[i].Size,
				ModTime: files[i].Time,
				IsDir:   files[i].Type == ftp.EntryTypeFolder,
			}
			return info, nil
		}
	}
	return nil, fs.ErrorObjectNotFound
}

func (f *Fs) mkdir(abspath string) error {
	_, err := f.getInfo(abspath)
	if err == fs.ErrorObjectNotFound {
		// fs.Debugf(f, "Trying to create directory %s", abspath)
		c, connErr := f.getFtpConnection()
		if connErr != nil {
			return errors.Wrap(connErr, "mkdir")
		}
		err = c.MakeDir(abspath)
		f.putFtpConnection(&c)
	}
	return err
}

// Mkdir creates the directory if it doesn't exist
func (f *Fs) Mkdir(dir string) (err error) {
	// defer fs.Trace(dir, "")("err=%v", &err)
	// This actually works as mkdir -p
	abspath := path.Join(f.root, dir)
	tokens := strings.Split(abspath, "/")
	curdir := ""
	for i := range tokens {
		curdir += tokens[i]
		if curdir == "" {
			continue
		}
		err := f.mkdir(curdir)
		if err != nil {
			return err
		}
		curdir += "/"
	}
	return nil
}

// Rmdir removes the directory (container, bucket) if empty
//
// Return an error if it doesn't exist or isn't empty
func (f *Fs) Rmdir(dir string) error {
	// This is actually a recursive remove directory
	c, err := f.getFtpConnection()
	if err != nil {
		return errors.Wrap(err, "Rmdir")
	}
	files, err := c.List(path.Join(f.root, dir))
	f.putFtpConnection(&c)
	if err != nil {
		return translateErrorDir(err)
	}
	for _, file := range files {
		if file.Type == ftp.EntryTypeFolder && file.Name != "." && file.Name != ".." {
			err = f.Rmdir(path.Join(dir, file.Name))
			if err != nil {
				return errors.Wrap(err, "rmdir")
			}
		}
	}
	c, err = f.getFtpConnection()
	if err != nil {
		return errors.Wrap(err, "Rmdir")
	}
	err = c.RemoveDir(path.Join(f.root, dir))
	f.putFtpConnection(&c)
	return translateErrorDir(err)
}

// ------------------------------------------------------------
<commit_msg>ftp: stop rmdir being recursive<commit_after>// Package ftp interfaces with FTP servers
package ftp

// FIXME Mover and DirMover are possible using c.Rename
// FIXME support conntimeout

import (
	"io"
	"net/textproto"
	"net/url"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/jlaffaye/ftp"
	"github.com/ncw/rclone/fs"
	"github.com/pkg/errors"
)

// Register with Fs
func init() {
	fs.Register(&fs.RegInfo{
		Name:        "ftp",
		Description: "FTP interface",
		NewFs:       NewFs,
		Options: []fs.Option{
			{
				Name: "username",
				Help: "Username",
			}, {
				Name:       "password",
				Help:       "Password",
				IsPassword: true,
			}, {
				Name: "url",
				Help: "FTP URL",
			},
		},
	})
}

// Fs represents a remote FTP server
type Fs struct {
	name     string       // name of this remote
	root     string       // the path we are working on if any
	features *fs.Features // optional features
	url      *url.URL
	user     string
	pass     string
	dialAddr string
	poolMu   sync.Mutex
	pool     []*ftp.ServerConn
}

// Object describes an FTP file
type Object struct {
	fs     *Fs
	remote string
	info   *FileInfo
}

// FileInfo is the metadata known about an FTP file
type FileInfo struct {
	Name    string
	Size    uint64
	ModTime time.Time
	IsDir   bool
}

// ------------------------------------------------------------

// Name of this fs
func (f *Fs) Name() string {
	return f.name
}

// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
	return f.root
}

// String returns a description of the FS
func (f *Fs) String() string {
	return f.url.String()
}

// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
	return f.features
}

// Open a new connection to the FTP server.
func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
	fs.Debugf(f, "Connecting to FTP server")
	c, err := ftp.DialTimeout(f.dialAddr, 30*time.Second)
	if err != nil {
		fs.Errorf(f, "Error while Dialing %s: %s", f.dialAddr, err)
		return nil, errors.Wrap(err, "ftpConnection Dial")
	}
	err = c.Login(f.user, f.pass)
	if err != nil {
		_ = c.Quit()
		fs.Errorf(f, "Error while Logging in into %s: %s", f.dialAddr, err)
		return nil, errors.Wrap(err, "ftpConnection Login")
	}
	return c, nil
}

// Get an FTP connection from the pool, or open a new one
func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
	f.poolMu.Lock()
	if len(f.pool) > 0 {
		c = f.pool[0]
		f.pool = f.pool[1:]
	}
	f.poolMu.Unlock()
	if c != nil {
		return c, nil
	}
	return f.ftpConnection()
}

// Return an FTP connection to the pool
//
// It nils the pointed to connection out so it can't be reused
func (f *Fs) putFtpConnection(c **ftp.ServerConn) {
	f.poolMu.Lock()
	f.pool = append(f.pool, *c)
	f.poolMu.Unlock()
}

// NewFs contstructs an Fs from the path, container:path
func NewFs(name, root string) (ff fs.Fs, err error) {
	// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
	URL := fs.ConfigFileGet(name, "url")
	user := fs.ConfigFileGet(name, "username")
	pass := fs.ConfigFileGet(name, "password")
	pass, err = fs.Reveal(pass)
	if err != nil {
		return nil, errors.Wrap(err, "NewFS decrypt password")
	}
	u, err := url.Parse(URL)
	if err != nil {
		return nil, errors.Wrap(err, "NewFS URL parse")
	}
	urlPath := strings.Trim(u.Path, "/")
	fullPath := root
	if urlPath != "" && !strings.HasPrefix("/", root) {
		fullPath = path.Join(u.Path, root)
	}
	root = fullPath
	dialAddr := u.Host
	if strings.IndexRune(dialAddr, ':') < 0 {
		dialAddr += ":21"
	}
	f := &Fs{
		name:     name,
		root:     root,
		url:      u,
		user:     user,
		pass:     <PASSWORD>,
		dialAddr: dialAddr,
	}
	f.features = (&fs.Features{}).Fill(f)
	// Make a connection and pool it to return errors early
	c, err := f.getFtpConnection()
	if err != nil {
		return nil, errors.Wrap(err, "NewFs")
	}
	f.putFtpConnection(&c)
	if root != "" {
		// Check to see if the root actually an existing file
		remote := path.Base(root)
		f.root = path.Dir(root)
		if f.root == "." {
			f.root = ""
		}
		_, err := f.NewObject(remote)
		if err != nil {
			if err == fs.ErrorObjectNotFound || errors.Cause(err) == fs.ErrorNotAFile {
				// File doesn't exist so return old f
				f.root = root
				return f, nil
			}
			return nil, err
		}
		// return an error with an fs which points to the parent
		return f, fs.ErrorIsFile
	}
	return f, err
}

// translateErrorFile turns FTP errors into rclone errors if possible for a file
func translateErrorFile(err error) error {
	switch errX := err.(type) {
	case *textproto.Error:
		switch errX.Code {
		case ftp.StatusFileUnavailable:
			err = fs.ErrorObjectNotFound
		}
	}
	return err
}

// translateErrorDir turns FTP errors into rclone errors if possible for a directory
func translateErrorDir(err error) error {
	switch errX := err.(type) {
	case *textproto.Error:
		switch errX.Code {
		case ftp.StatusFileUnavailable:
			err = fs.ErrorDirNotFound
		}
	}
	return err
}

// NewObject finds the Object at remote.  If it can't be found
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(remote string) (o fs.Object, err error) {
	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
	fullPath := path.Join(f.root, remote)
	dir := path.Dir(fullPath)
	base := path.Base(fullPath)

	c, err := f.getFtpConnection()
	if err != nil {
		return nil, errors.Wrap(err, "NewObject")
	}
	files, err := c.List(dir)
	f.putFtpConnection(&c)
	if err != nil {
		return nil, translateErrorFile(err)
	}
	for i, file := range files {
		if file.Type != ftp.EntryTypeFolder && file.Name == base {
			o := &Object{
				fs:     f,
				remote: remote,
			}
			info := &FileInfo{
				Name:    remote,
				Size:    files[i].Size,
				ModTime: files[i].Time,
			}
			o.info = info

			return o, nil
		}
	}
	return nil, fs.ErrorObjectNotFound
}

func (f *Fs) list(out fs.ListOpts, dir string, curlevel int) {
	// defer fs.Trace(dir, "curlevel=%d", curlevel)("")
	c, err := f.getFtpConnection()
	if err != nil {
		out.SetError(errors.Wrap(err, "list"))
		return
	}
	files, err := c.List(path.Join(f.root, dir))
	f.putFtpConnection(&c)
	if err != nil {
		out.SetError(translateErrorDir(err))
		return
	}
	for i := range files {
		object := files[i]
		newremote := path.Join(dir, object.Name)
		switch object.Type {
		case ftp.EntryTypeFolder:
			if object.Name == "." || object.Name == ".." {
				continue
			}
			if out.IncludeDirectory(newremote) {
				d := &fs.Dir{
					Name:  newremote,
					When:  object.Time,
					Bytes: 0,
					Count: -1,
				}
				if curlevel < out.Level() {
					f.list(out, path.Join(dir, object.Name), curlevel+1)
				}
				if out.AddDir(d) {
					return
				}
			}
		default:
			o := &Object{
				fs:     f,
				remote: newremote,
			}
			info := &FileInfo{
				Name:    newremote,
				Size:    object.Size,
				ModTime: object.Time,
			}
			o.info = info
			if out.Add(o) {
				return
			}
		}
	}
}

// List the objects and directories of the Fs starting from dir
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound (using out.SetError())
// if the directory isn't found.
//
// Fses must support recursion levels of fs.MaxLevel and 1.
// They may return ErrorLevelNotSupported otherwise.
func (f *Fs) List(out fs.ListOpts, dir string) {
	// defer fs.Trace(dir, "")("")
	f.list(out, dir, 1)
	out.Finished()
}

// Hashes are not supported
func (f *Fs) Hashes() fs.HashSet {
	return 0
}

// Precision shows Modified Time not supported
func (f *Fs) Precision() time.Duration {
	return fs.ModTimeNotSupported
}

// Put in to the remote path with the modTime given of the given size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
	// fs.Debugf(f, "Trying to put file %s", src.Remote())
	o := &Object{
		fs:     f,
		remote: src.Remote(),
	}
	err := o.Update(in, src)
	return o, err
}

// getInfo reads the FileInfo for a path
func (f *Fs) getInfo(remote string) (fi *FileInfo, err error) {
	// defer fs.Trace(remote, "")("fi=%v, err=%v", &fi, &err)
	dir := path.Dir(remote)
	base := path.Base(remote)

	c, err := f.getFtpConnection()
	if err != nil {
		return nil, errors.Wrap(err, "getInfo")
	}
	files, err := c.List(dir)
	f.putFtpConnection(&c)
	if err != nil {
		return nil, translateErrorFile(err)
	}

	for i := range files {
		if files[i].Name == base {
			info := &FileInfo{
				Name:    remote,
				Size:    files[i].Size,
				ModTime: files[i].Time,
				IsDir:   files[i].Type == ftp.EntryTypeFolder,
			}
			return info, nil
		}
	}
	return nil, fs.ErrorObjectNotFound
}

func (f *Fs) mkdir(abspath string) error {
	_, err := f.getInfo(abspath)
	if err == fs.ErrorObjectNotFound {
		// fs.Debugf(f, "Trying to create directory %s", abspath)
		c, connErr := f.getFtpConnection()
		if connErr != nil {
			return errors.Wrap(connErr, "mkdir")
		}
		err = c.MakeDir(abspath)
		f.putFtpConnection(&c)
	}
	return err
}

// Mkdir creates the directory if it doesn't exist
func (f *Fs) Mkdir(dir string) (err error) {
	// defer fs.Trace(dir, "")("err=%v", &err)
	// This actually works as mkdir -p
	abspath := path.Join(f.root, dir)
	tokens := strings.Split(abspath, "/")
	curdir := ""
	for i := range tokens {
		curdir += tokens[i]
		if curdir == "" {
			continue
		}
		err := f.mkdir(curdir)
		if err != nil {
			return err
		}
		curdir += "/"
	}
	return nil
}

// Rmdir removes the directory (container, bucket) if empty
//
// Return an error if it doesn't exist or isn't empty
func (f *Fs) Rmdir(dir string) error {
	c, err := f.getFtpConnection()
	if err != nil {
		return errors.Wrap(translateErrorFile(err), "Rmdir")
	}
	err = c.RemoveDir(path.Join(f.root, dir))
	f.putFtpConnection(&c)
	return translateErrorDir(err)
}

// ------------------------------------------------------------

{'fraction_non_alphanumeric': 0.15656193145498354, 'fraction_numerical': 0.0015734867482912918, 'mean_word_length': 2.833741753063148, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7072855', 'n_tokens_mistral': 8082, 'n_tokens_neox': 7375, 'n_words': 2212}