<commit_before>// Test rclone by doing real transactions to a storage provider to and
// from the local disk

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"path"
	"strings"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/ogier/pflag"

	// Active file systems
	_ "github.com/ncw/rclone/drive"
	_ "github.com/ncw/rclone/local"
	_ "github.com/ncw/rclone/s3"
	_ "github.com/ncw/rclone/swift"
)

// Globals
var (
	localName, remoteName string
	version               = pflag.BoolP("version", "V", false, "Print the version number")
)

// Represents an item for checking
type Item struct {
	Path    string
	Md5sum  string
	ModTime time.Time
	Size    int64
}

// Represents all items for checking
type Items struct {
	byName map[string]*Item
	items  []Item
}

// Make an Items
func NewItems(items []Item) *Items {
	is := &Items{
		byName: make(map[string]*Item),
		items:  items,
	}
	// Fill up byName
	for i := range items {
		is.byName[items[i].Path] = &items[i]
	}
	return is
}

// Check off an item
func (is *Items) Find(obj fs.Object) {
	i, ok := is.byName[obj.Remote()]
	if !ok {
		log.Fatalf("Unexpected file %q", obj.Remote())
	}
	delete(is.byName, obj.Remote())
	// Check attributes
	Md5sum, err := obj.Md5sum()
	if err != nil {
		log.Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err)
	}
	if i.Md5sum != Md5sum {
		log.Fatalf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum)
	}
	if i.Size != obj.Size() {
		log.Fatalf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size())
	}
	// check the mod time to the given precision
	modTime := obj.ModTime()
	dt := modTime.Sub(i.ModTime)
	if dt >= fs.Config.ModifyWindow || dt <= -fs.Config.ModifyWindow {
		log.Fatalf("%s: Modification time difference too big |%s| > %s (%s vs %s)", obj.Remote(), dt, fs.Config.ModifyWindow, modTime, i.ModTime)
	}

}

// Check all done
func (is *Items) Done() {
	if len(is.byName) != 0 {
		for name := range is.byName {
			log.Printf("Not found %q", name)
		}
		log.Fatalf("%d objects not found", len(is.byName))
	}
}

// Checks the fs to see if it has the expected contents
func CheckListing(f fs.Fs, items []Item) {
	is := NewItems(items)
	for obj := range f.List() {
		is.Find(obj)
	}
	is.Done()
}

// Parse a time string or explode
func Time(timeString string) time.Time {
	t, err := time.Parse(time.RFC3339Nano, timeString)
	if err != nil {
		log.Fatalf("Failed to parse time %q: %v", timeString, err)
	}
	return t
}

// Write a file
func WriteFile(filePath, content string, t time.Time) {
	// FIXME make directories?
	filePath = path.Join(localName, filePath)
	err := ioutil.WriteFile(filePath, []byte(content), 0600)
	if err != nil {
		log.Fatalf("Failed to write file %q: %v", filePath, err)
	}
	err = os.Chtimes(filePath, t, t)
	if err != nil {
		log.Fatalf("Failed to chtimes file %q: %v", filePath, err)
	}
}

// Create a random string
func RandomString(n int) string {
	source := "abcdefghijklmnopqrstuvwxyz0123456789"
	out := make([]byte, n)
	for i := range out {
		out[i] = source[rand.Intn(len(source))]
	}
	return string(out)
}

func TestMkdir(flocal, fremote fs.Fs) {
	err := fs.Mkdir(fremote)
	if err != nil {
		log.Fatalf("Mkdir failed: %v", err)
	}
	items := []Item{}
	CheckListing(flocal, items)
	CheckListing(fremote, items)
}

var t1 = Time("2001-02-03T04:05:06.499999999Z")
var t2 = Time("2011-12-25T12:59:59.123456789Z")
var t3 = Time("2011-12-30T12:59:59.000000000Z")

func TestCopy(flocal, fremote fs.Fs) {
	WriteFile("empty space", "", t1)

	// Check dry run is working
	log.Printf("Copy with --dry-run")
	fs.Config.DryRun = true
	err := fs.Sync(fremote, flocal, false)
	fs.Config.DryRun = false
	if err != nil {
		log.Fatalf("Copy failed: %v", err)
	}

	items := []Item{
		{Path: "empty space", Size: 0, ModTime: t1, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
	}

	CheckListing(flocal, items)
	CheckListing(fremote, []Item{})

	// Now without dry run

	log.Printf("Copy")
	err = fs.Sync(fremote, flocal, false)
	if err != nil {
		log.Fatalf("Copy failed: %v", err)
	}

	CheckListing(flocal, items)
	CheckListing(fremote, items)
}

func TestSync(flocal, fremote fs.Fs) {
	log.Printf("Sync after changing file modtime only")
	err := os.Chtimes(localName+"/empty space", t2, t2)
	if err != nil {
		log.Fatalf("Chtimes failed: %v", err)
	}
	err = fs.Sync(fremote, flocal, true)
	if err != nil {
		log.Fatalf("Sync failed: %v", err)
	}
	items := []Item{
		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
	}
	CheckListing(flocal, items)
	CheckListing(fremote, items)

	// ------------------------------------------------------------

	log.Printf("Sync after adding a file")
	WriteFile("potato", "------------------------------------------------------------", t3)<commit_msg>rclonetest: check sub directory and downloads<commit_after>// Test rclone by doing real transactions to a storage provider to and
// from the local disk

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"path"
	"strings"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/ogier/pflag"

	// Active file systems
	_ "github.com/ncw/rclone/drive"
	_ "github.com/ncw/rclone/local"
	_ "github.com/ncw/rclone/s3"
	_ "github.com/ncw/rclone/swift"
)

// Globals
var (
	localName, remoteName string
	version               = pflag.BoolP("version", "V", false, "Print the version number")
)

// Represents an item for checking
type Item struct {
	Path    string
	Md5sum  string
	ModTime time.Time
	Size    int64
}

// Represents all items for checking
type Items struct {
	byName map[string]*Item
	items  []Item
}

// Make an Items
func NewItems(items []Item) *Items {
	is := &Items{
		byName: make(map[string]*Item),
		items:  items,
	}
	// Fill up byName
	for i := range items {
		is.byName[items[i].Path] = &items[i]
	}
	return is
}

// Check off an item
func (is *Items) Find(obj fs.Object) {
	i, ok := is.byName[obj.Remote()]
	if !ok {
		log.Fatalf("Unexpected file %q", obj.Remote())
	}
	delete(is.byName, obj.Remote())
	// Check attributes
	Md5sum, err := obj.Md5sum()
	if err != nil {
		log.Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err)
	}
	if i.Md5sum != Md5sum {
		log.Fatalf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum)
	}
	if i.Size != obj.Size() {
		log.Fatalf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size())
	}
	// check the mod time to the given precision
	modTime := obj.ModTime()
	dt := modTime.Sub(i.ModTime)
	if dt >= fs.Config.ModifyWindow || dt <= -fs.Config.ModifyWindow {
		log.Fatalf("%s: Modification time difference too big |%s| > %s (%s vs %s)", obj.Remote(), dt, fs.Config.ModifyWindow, modTime, i.ModTime)
	}

}

// Check all done
func (is *Items) Done() {
	if len(is.byName) != 0 {
		for name := range is.byName {
			log.Printf("Not found %q", name)
		}
		log.Fatalf("%d objects not found", len(is.byName))
	}
}

// Checks the fs to see if it has the expected contents
func CheckListing(f fs.Fs, items []Item) {
	is := NewItems(items)
	for obj := range f.List() {
		is.Find(obj)
	}
	is.Done()
}

// Parse a time string or explode
func Time(timeString string) time.Time {
	t, err := time.Parse(time.RFC3339Nano, timeString)
	if err != nil {
		log.Fatalf("Failed to parse time %q: %v", timeString, err)
	}
	return t
}

// Write a file
func WriteFile(filePath, content string, t time.Time) {
	// FIXME make directories?
	filePath = path.Join(localName, filePath)
	dirPath := path.Dir(filePath)
	err := os.MkdirAll(dirPath, 0770)
	if err != nil {
		log.Fatalf("Failed to make directories %q: %v", dirPath, err)
	}
	err = ioutil.WriteFile(filePath, []byte(content), 0600)
	if err != nil {
		log.Fatalf("Failed to write file %q: %v", filePath, err)
	}
	err = os.Chtimes(filePath, t, t)
	if err != nil {
		log.Fatalf("Failed to chtimes file %q: %v", filePath, err)
	}
}

// Create a random string
func RandomString(n int) string {
	source := "abcdefghijklmnopqrstuvwxyz0123456789"
	out := make([]byte, n)
	for i := range out {
		out[i] = source[rand.Intn(len(source))]
	}
	return string(out)
}

func TestMkdir(flocal, fremote fs.Fs) {
	err := fs.Mkdir(fremote)
	if err != nil {
		log.Fatalf("Mkdir failed: %v", err)
	}
	items := []Item{}
	CheckListing(flocal, items)
	CheckListing(fremote, items)
}

var t1 = Time("2001-02-03T04:05:06.499999999Z")
var t2 = Time("2011-12-25T12:59:59.123456789Z")
var t3 = Time("2011-12-30T12:59:59.000000000Z")

func TestCopy(flocal, fremote fs.Fs) {
	WriteFile("sub dir/hello world", "hello world", t1)

	// Check dry run is working
	log.Printf("Copy with --dry-run")
	fs.Config.DryRun = true
	err := fs.Sync(fremote, flocal, false)
	fs.Config.DryRun = false
	if err != nil {
		log.Fatalf("Copy failed: %v", err)
	}

	items := []Item{
		{Path: "sub dir/hello world", Size: 11, ModTime: t1, Md5sum: "5eb63bbbe01eeed093cb22bb8f5acdc3"},
	}

	CheckListing(flocal, items)
	CheckListing(fremote, []Item{})

	// Now without dry run

	log.Printf("Copy")
	err = fs.Sync(fremote, flocal, false)
	if err != nil {
		log.Fatalf("Copy failed: %v", err)
	}

	CheckListing(flocal, items)
	CheckListing(fremote, items)

	// Now delete the local file and download it

	err = os.Remove(localName + "/sub dir/hello world")
	if err != nil {
		log.Fatalf("Remove failed: %v", err)
	}

	CheckListing(flocal, []Item{})
	CheckListing(fremote, items)

	log.Printf("Copy - redownload")
	err = fs.Sync(flocal, fremote, false)
	if err != nil {
		log.Fatalf("Copy failed: %v", err)
	}

	CheckListing(flocal, items)
	CheckListing(fremote, items)

	// Clean the directory
	cleanTempDir()
}

func TestSync(flocal, fremote fs.Fs) {
	WriteFile("empty space", "", t1)

	log.Printf("Sync after changing file modtime only")
	err := os.Chtimes(localName+"/empty space", t2, t2)
	if err != nil {
		log.Fatalf("Chtimes failed: %v", err)
	}
	err = fs.Sync(fremote, flocal, true)
	if err != nil {
		log.Fatalf("Sync failed: %v", err)
	}
	items := []Item{
		{Path: "empty space", Size: 0, ModTime: t2, Md5sum: "d41d8cd98f00b204e9800998ecf8427e"},
	}
	CheckListing(flocal, items)
	CheckListing(fremote, items)

	// ------------------------------------------------------------

	log.Printf("Sync after adding a file")
	WriteFile("potato", "------------------------------------------------------------", t3)
{'fraction_non_alphanumeric': 0.18017843289371605, 'fraction_numerical': 0.02986811481768813, 'mean_word_length': 3.3625211505922166, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5867023', 'n_tokens_mistral': 4225, 'n_tokens_neox': 3988, 'n_words': 1044}