<commit_before>// Package webdav provides an interface to the Webdav
// object storage system.
package webdav

// SetModTime might be possible
// https://stackoverflow.com/questions/3579608/webdav-can-a-client-modify-the-mtime-of-a-file
// ...support for a PROPSET to lastmodified (mind the missing get) which does the utime() call might be an option.
// For example the ownCloud WebDAV server does it that way.

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"

	"github.com/ncw/rclone/backend/webdav/api"
	"github.com/ncw/rclone/backend/webdav/odrvcookie"
	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/config/configmap"
	"github.com/ncw/rclone/fs/config/configstruct"
	"github.com/ncw/rclone/fs/config/obscure"
	"github.com/ncw/rclone/fs/fserrors"
	"github.com/ncw/rclone/fs/fshttp"
	"github.com/ncw/rclone/fs/hash"
	"github.com/ncw/rclone/lib/pacer"
	"github.com/ncw/rclone/lib/rest"
	"github.com/pkg/errors"
)

const (
	minSleep      = 10 * time.Millisecond
	maxSleep      = 2 * time.Second
	decayConstant = 2   // bigger for slower decay, exponential
	defaultDepth  = "1" // depth for PROPFIND
)

// Register with Fs
func init() {
	fs.Register(&fs.RegInfo{
		Name:        "webdav",
		Description: "Webdav",
		NewFs:       NewFs,
		Options: []fs.Option{{
			Name:     "url",
			Help:     "URL of http host to connect to",
			Required: true,
			Examples: []fs.OptionExample{{
				Value: "https://example.com",
				Help:  "Connect to example.com",
			}},
		}, {
			Name: "vendor",
			Help: "Name of the Webdav site/service/software you are using",
			Examples: []fs.OptionExample{{
				Value: "nextcloud",
				Help:  "Nextcloud",
			}, {
				Value: "owncloud",
				Help:  "Owncloud",
			}, {
				Value: "sharepoint",
				Help:  "Sharepoint",
			}, {
				Value: "other",
				Help:  "Other site/service or software",
			}},
		}, {
			Name: "user",
			Help: "User name",
		}, {
			Name:       "pass",
			Help:       "Password.",
			IsPassword: true,
		}, {
			Name: "bearer_token",
			Help: "Bearer token instead of user/pass (eg a Macaroon)",
		}},
	})
}

// Options defines the configuration for this backend
type Options struct {
	URL         string `config:"url"`
	Vendor      string `config:"vendor"`
	User        string `config:"user"`
	Pass        string `config:"pass"`
	BearerToken string `config:"bearer_token"`
}

// Fs represents a remote webdav
type Fs struct {
	name               string        // name of this remote
	root               string        // the path we are working on
	opt                Options       // parsed options
	features           *fs.Features  // optional features
	endpoint           *url.URL      // URL of the host
	endpointURL        string        // endpoint as a string
	srv                *rest.Client  // the connection to the one drive server
	pacer              *fs.Pacer     // pacer for API calls
	precision          time.Duration // mod time precision
	canStream          bool          // set if can stream
	useOCMtime         bool          // set if can use X-OC-Mtime
	retryWithZeroDepth bool          // some vendors (sharepoint) won't list files when Depth is 1 (our default)
	hasChecksums       bool          // set if can use owncloud style checksums
}

// Object describes a webdav object
//
// Will definitely have info but maybe not meta
type Object struct {
	fs          *Fs       // what this object is part of
	remote      string    // The remote path
	hasMetaData bool      // whether info below has been set
	size        int64     // size of the object
	modTime     time.Time // modification time of the object
	sha1        string    // SHA-1 of the object content if known
	md5         string    // MD5 of the object content if known
}

// ------------------------------------------------------------

// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
	return f.name
}

// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
	return f.root
}

// String converts this Fs to a string
func (f *Fs) String() string {
	return fmt.Sprintf("webdav root '%s'", f.root)
}

// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
	return f.features
}

// retryErrorCodes is a slice of error codes that we will retry
var retryErrorCodes = []int{
	429, // Too Many Requests.
	500, // Internal Server Error
	502, // Bad Gateway
	503, // Service Unavailable
	504, // Gateway Timeout
	509, // Bandwidth Limit Exceeded
}

// shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried.  It returns the err as a convenience
func shouldRetry(resp *http.Response, err error) (bool, error) {
	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
}

// itemIsDir returns true if the item is a directory
//
// When a client sees a resourcetype it doesn't recognize it should
// assume it is a regular non-collection resource.  [WebDav book by
// <NAME> ch 7.5.8 p170]
func itemIsDir(item *api.Response) bool {
	if t := item.Props.Type; t != nil {
		if t.Space == "DAV:" && t.Local == "collection" {
			return true
		}
		fs.Debugf(nil, "Unknown resource type %q/%q on %q", t.Space, t.Local, item.Props.Name)
	}<commit_msg>webdav: retry on 423 Locked errors #3263<commit_after>// Package webdav provides an interface to the Webdav
// object storage system.
package webdav

// SetModTime might be possible
// https://stackoverflow.com/questions/3579608/webdav-can-a-client-modify-the-mtime-of-a-file
// ...support for a PROPSET to lastmodified (mind the missing get) which does the utime() call might be an option.
// For example the ownCloud WebDAV server does it that way.

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"

	"github.com/ncw/rclone/backend/webdav/api"
	"github.com/ncw/rclone/backend/webdav/odrvcookie"
	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/config/configmap"
	"github.com/ncw/rclone/fs/config/configstruct"
	"github.com/ncw/rclone/fs/config/obscure"
	"github.com/ncw/rclone/fs/fserrors"
	"github.com/ncw/rclone/fs/fshttp"
	"github.com/ncw/rclone/fs/hash"
	"github.com/ncw/rclone/lib/pacer"
	"github.com/ncw/rclone/lib/rest"
	"github.com/pkg/errors"
)

const (
	minSleep      = 10 * time.Millisecond
	maxSleep      = 2 * time.Second
	decayConstant = 2   // bigger for slower decay, exponential
	defaultDepth  = "1" // depth for PROPFIND
)

// Register with Fs
func init() {
	fs.Register(&fs.RegInfo{
		Name:        "webdav",
		Description: "Webdav",
		NewFs:       NewFs,
		Options: []fs.Option{{
			Name:     "url",
			Help:     "URL of http host to connect to",
			Required: true,
			Examples: []fs.OptionExample{{
				Value: "https://example.com",
				Help:  "Connect to example.com",
			}},
		}, {
			Name: "vendor",
			Help: "Name of the Webdav site/service/software you are using",
			Examples: []fs.OptionExample{{
				Value: "nextcloud",
				Help:  "Nextcloud",
			}, {
				Value: "owncloud",
				Help:  "Owncloud",
			}, {
				Value: "sharepoint",
				Help:  "Sharepoint",
			}, {
				Value: "other",
				Help:  "Other site/service or software",
			}},
		}, {
			Name: "user",
			Help: "User name",
		}, {
			Name:       "pass",
			Help:       "Password.",
			IsPassword: true,
		}, {
			Name: "bearer_token",
			Help: "Bearer token instead of user/pass (eg a Macaroon)",
		}},
	})
}

// Options defines the configuration for this backend
type Options struct {
	URL         string `config:"url"`
	Vendor      string `config:"vendor"`
	User        string `config:"user"`
	Pass        string `config:"pass"`
	BearerToken string `config:"bearer_token"`
}

// Fs represents a remote webdav
type Fs struct {
	name               string        // name of this remote
	root               string        // the path we are working on
	opt                Options       // parsed options
	features           *fs.Features  // optional features
	endpoint           *url.URL      // URL of the host
	endpointURL        string        // endpoint as a string
	srv                *rest.Client  // the connection to the one drive server
	pacer              *fs.Pacer     // pacer for API calls
	precision          time.Duration // mod time precision
	canStream          bool          // set if can stream
	useOCMtime         bool          // set if can use X-OC-Mtime
	retryWithZeroDepth bool          // some vendors (sharepoint) won't list files when Depth is 1 (our default)
	hasChecksums       bool          // set if can use owncloud style checksums
}

// Object describes a webdav object
//
// Will definitely have info but maybe not meta
type Object struct {
	fs          *Fs       // what this object is part of
	remote      string    // The remote path
	hasMetaData bool      // whether info below has been set
	size        int64     // size of the object
	modTime     time.Time // modification time of the object
	sha1        string    // SHA-1 of the object content if known
	md5         string    // MD5 of the object content if known
}

// ------------------------------------------------------------

// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
	return f.name
}

// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
	return f.root
}

// String converts this Fs to a string
func (f *Fs) String() string {
	return fmt.Sprintf("webdav root '%s'", f.root)
}

// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
	return f.features
}

// retryErrorCodes is a slice of error codes that we will retry
var retryErrorCodes = []int{
	423, // Locked
	429, // Too Many Requests.
	500, // Internal Server Error
	502, // Bad Gateway
	503, // Service Unavailable
	504, // Gateway Timeout
	509, // Bandwidth Limit Exceeded
}

// shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried.  It returns the err as a convenience
func shouldRetry(resp *http.Response, err error) (bool, error) {
	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
}

// itemIsDir returns true if the item is a directory
//
// When a client sees a resourcetype it doesn't recognize it should
// assume it is a regular non-collection resource.  [WebDav book by
// <NAME> ch 7.5.8 p170]
func itemIsDir(item *api.Response) bool {
	if t := item.Props.Type; t != nil {
		if t.Space == "DAV:" && t.Local == "collection" {
			return true
		}
		fs.Debugf(nil, "Unknown resource type %q/%q on %q", t.Space, t.Local, item.Props.Name)
	}
{'fraction_non_alphanumeric': 0.12669126691266913, 'fraction_numerical': 0.00908316775475447, 'mean_word_length': 2.4486133768352367, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5751676', 'n_tokens_mistral': 3659, 'n_tokens_neox': 3359, 'n_words': 1126}