<commit_before>// hblsync -- High Bandwidth & Latency directory sync tool
//
// Copyright 2010 <NAME>
// <EMAIL>
//
// Usage: hblsync <srcdir> <dstdir>
//

package main

import "container/vector"
import "exp/iterable"
import "fmt"
import "flag"
import "os"

var delete *bool = flag.Bool("delete", false, "delete mode (takes 1 argument)")

func usage() {
	os.Stderr.WriteString("Usage: hblsync <src_dir> <dst_dir>\n")
	os.Stderr.WriteString("       hblsync --delete <dir>\n")
	os.Exit(1)
}

func main() {
	flag.Parse()
	resultChan := make(chan SyncStats)

	// Delete mode
	if (*delete) {
		if (flag.NArg() != 1) { usage() }
		go RemoveAll(flag.Arg(0), resultChan)
		fmt.Println("Results:", <-resultChan)
		return
	}

	// Sync mode
	if (flag.NArg() != 2) {
		usage();
	}

	srcDir, dstDir := flag.Arg(0), flag.Arg(1)
	ensureDirectory("source", srcDir)
	ensureDirectory("destination", dstDir)

	go SyncDirectories(srcDir, dstDir, resultChan)
	fmt.Println("Results:", <-resultChan)
}

func ensureDirectory(which string, dirName string) {
	stat, e := os.Stat(dirName)
	if e != nil {
		fmt.Fprintf(os.Stderr, "Error stat'ing %s directory: %s\n",
			which, e.String())
		os.Exit(1)
	}
	if !stat.IsDirectory() {
		fmt.Fprintf(os.Stderr, "%s directory isn't a directory: %s\n",
			which, dirName)
		os.Exit(1)
	}
}

type SyncStats struct {
	ErrorCount int

	FilesGood int   // no change needed
	DirsGood int    // already existed (contents might be wrong)

	DirsCreated int
	FilesCreated int

	FilesReplaced int  // existed, but wrong. doesn't count in FilesDeleted
	DirsDeleted int
	FilesDeleted int
}

func (stats *SyncStats) incrementBy(delta *SyncStats) {
	stats.ErrorCount += delta.ErrorCount
	stats.DirsCreated += delta.DirsCreated
	stats.DirsDeleted += delta.DirsDeleted
	stats.DirsGood += delta.DirsGood
	stats.FilesCreated += delta.FilesCreated
	stats.FilesReplaced += delta.FilesReplaced
	stats.FilesDeleted += delta.FilesDeleted
	stats.FilesGood += delta.FilesGood
}

type outstandingOps struct {
	v vector.Vector;  // of chan SyncStats
}

func (ops *outstandingOps) new() chan SyncStats {
	ch := make(chan SyncStats)
	ops.v.Push(ch)
	return ch
}

// Wait for all outstandin operations, summing the total into outStats
func (ops *outstandingOps) wait(outStats *SyncStats) {
	for _, value := range ops.v {
		ch := value.(chan SyncStats)
		subStats := <-ch
		outStats.incrementBy(&subStats)
	}
}

func readDirnames(dir string, outNames *[]string, ok chan bool) {
	fd, err := os.Open(dir, os.O_RDONLY, 0)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error reading dirnames: %v\n", err)
		ok <- false
	}
	defer fd.Close()
	names, err := fd.Readdirnames(-1)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error reading dirnames: %v\n", err)
		ok <- false
	} else {
		*outNames = names
		ok <- true
	}
}

func stringSet(items []string) map[string]bool {
	var set = map[string]bool {}
	for _, name := range items {
		set[name] = true
	}
	return set
}

func makeMapLookupTest(someMap map[string]bool) func(entry interface{}) bool {
	return func(entry interface{}) bool {
                name := entry.(string)
		if someMap[name] {
			return true
		}
		return false
	}
}

func copyFile(srcName string, dstName string, out chan SyncStats) {
	fmt.Printf("Copying files not yet implemented-- %s -> %s\n",
		srcName, dstName)
	// TODO: implement
	sendError(out)
}

func copyDirectory(srcName string, stat *os.FileInfo, dstName string,
	           out chan SyncStats) {
	stats := new(SyncStats)
	defer func() {
		out <- *stats
	}()

	err := os.Mkdir(dstName, stat.Permission())
	if err != nil {
		stats.ErrorCount++
		return
	}
	stats.DirsCreated++

	ops := new(outstandingOps)
	go SyncDirectories(srcName, dstName, ops.new())
	ops.wait(stats)
}

func sendError(out chan SyncStats) {
	stats := new(SyncStats)
	stats.ErrorCount++
	out <- *stats
}

func Copy(srcName string, dstName string, out chan SyncStats) {
	srcstat, serr := os.Lstat(srcName)
	if serr != nil {
                fmt.Fprintf(os.Stderr, "Can't stat source %s: %v", srcName, serr)
		sendError(out)
		return
	}

	switch {
	case srcstat.IsRegular():
		copyFile(srcName, dstName, out)
	case srcstat.IsDirectory():
		copyDirectory(srcName, srcstat, dstName, out)
	default:<commit_msg>copying regular files<commit_after>// hblsync -- High Bandwidth & Latency directory sync tool
//
// Copyright 2010 <NAME>
// <EMAIL>
//
// Usage: hblsync <srcdir> <dstdir>
//

package main

import "container/vector"
import "exp/iterable"
import "fmt"
import "flag"
import "os"

var delete *bool = flag.Bool("delete", false, "delete mode (takes 1 argument)")

func usage() {
	os.Stderr.WriteString("Usage: hblsync <src_dir> <dst_dir>\n")
	os.Stderr.WriteString("       hblsync --delete <dir>\n")
	os.Exit(1)
}

func main() {
	flag.Parse()
	resultChan := make(chan SyncStats)

	// Delete mode
	if (*delete) {
		if (flag.NArg() != 1) { usage() }
		go RemoveAll(flag.Arg(0), resultChan)
		fmt.Println("Results:", <-resultChan)
		return
	}

	// Sync mode
	if (flag.NArg() != 2) {
		usage();
	}

	srcDir, dstDir := flag.Arg(0), flag.Arg(1)
	ensureDirectory("source", srcDir)
	ensureDirectory("destination", dstDir)

	go SyncDirectories(srcDir, dstDir, resultChan)
	fmt.Println("Results:", <-resultChan)
}

func ensureDirectory(which string, dirName string) {
	stat, e := os.Stat(dirName)
	if e != nil {
		fmt.Fprintf(os.Stderr, "Error stat'ing %s directory: %s\n",
			which, e.String())
		os.Exit(1)
	}
	if !stat.IsDirectory() {
		fmt.Fprintf(os.Stderr, "%s directory isn't a directory: %s\n",
			which, dirName)
		os.Exit(1)
	}
}

type SyncStats struct {
	ErrorCount int

	FilesGood int   // no change needed
	DirsGood int    // already existed (contents might be wrong)

	DirsCreated int
	FilesCreated int

	FilesReplaced int  // existed, but wrong. doesn't count in FilesDeleted
	DirsDeleted int
	FilesDeleted int
}

func (stats *SyncStats) incrementBy(delta *SyncStats) {
	stats.ErrorCount += delta.ErrorCount
	stats.DirsCreated += delta.DirsCreated
	stats.DirsDeleted += delta.DirsDeleted
	stats.DirsGood += delta.DirsGood
	stats.FilesCreated += delta.FilesCreated
	stats.FilesReplaced += delta.FilesReplaced
	stats.FilesDeleted += delta.FilesDeleted
	stats.FilesGood += delta.FilesGood
}

type outstandingOps struct {
	v vector.Vector;  // of chan SyncStats
}

func (ops *outstandingOps) new() chan SyncStats {
	ch := make(chan SyncStats)
	ops.v.Push(ch)
	return ch
}

// Wait for all outstandin operations, summing the total into outStats
func (ops *outstandingOps) wait(outStats *SyncStats) {
	for _, value := range ops.v {
		ch := value.(chan SyncStats)
		subStats := <-ch
		outStats.incrementBy(&subStats)
	}
}

func readDirnames(dir string, outNames *[]string, ok chan bool) {
	fd, err := os.Open(dir, os.O_RDONLY, 0)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error reading dirnames: %v\n", err)
		ok <- false
	}
	defer fd.Close()
	names, err := fd.Readdirnames(-1)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error reading dirnames: %v\n", err)
		ok <- false
	} else {
		*outNames = names
		ok <- true
	}
}

func stringSet(items []string) map[string]bool {
	var set = map[string]bool {}
	for _, name := range items {
		set[name] = true
	}
	return set
}

func makeMapLookupTest(someMap map[string]bool) func(entry interface{}) bool {
	return func(entry interface{}) bool {
                name := entry.(string)
		if someMap[name] {
			return true
		}
		return false
	}
}

func copyRegularFile(srcName string, stat *os.FileInfo, dstName string,
	             out chan SyncStats) {
	stats := new(SyncStats)
	defer func() { out <- *stats }()

	outfd, err := os.Open(dstName, os.O_CREATE|os.O_EXCL|os.O_WRONLY,
		stat.Permission())
	if err != nil {
		fmt.Fprintf(os.Stderr,
			"Error opening copy output file %s: %s\n",
			dstName, err)
		stats.ErrorCount++
		return
	}
	defer outfd.Close()

	infd, err := os.Open(srcName, os.O_RDONLY, 0)
	if err != nil {
                fmt.Fprintf(os.Stderr,
                        "Error opening copy source file %s: %s\n",
                        srcName, err)
                stats.ErrorCount++
                return
	}
	defer infd.Close()

	const BUF_SIZE = 1024 * 256
	buf := make([]byte, BUF_SIZE)
	bytesRemain := stat.Size
	for (bytesRemain > 0) {
		n, err := infd.Read(buf)
		switch {
		case n == 0:
			stats.FilesCreated++
			return
		case n < 0:
			stats.ErrorCount++
			fmt.Fprintf(os.Stderr, "Error copying file %s in read: %s",
				srcName, err)
			return
		default:
			outN, err := outfd.Write(buf[0 : n])
			if err != nil || outN != n {
				fmt.Fprintf(os.Stderr, "Error copying file %s in write: %s",
					srcName, err)
				return
			}
			bytesRemain -= int64(outN)
		}
	}
}

func copyDirectory(srcName string, stat *os.FileInfo, dstName string,
	           out chan SyncStats) {
	stats := new(SyncStats)
	defer func() { out <- *stats }()

	err := os.Mkdir(dstName, stat.Permission())
	if err != nil {
		stats.ErrorCount++
		return
	}
	stats.DirsCreated++

	ops := new(outstandingOps)
	go SyncDirectories(srcName, dstName, ops.new())
	ops.wait(stats)
}

func sendError(out chan SyncStats) {
	stats := new(SyncStats)
	stats.ErrorCount++
	out <- *stats
}

func Copy(srcName string, dstName string, out chan SyncStats) {
	srcStat, serr := os.Lstat(srcName)
	if serr != nil {
                fmt.Fprintf(os.Stderr, "Can't stat source %s: %v", srcName, serr)
		sendError(out)
		return
	}

	switch {
	case srcStat.IsRegular():
		copyRegularFile(srcName, srcStat, dstName, out)
	case srcStat.IsDirectory():
		copyDirectory(srcName, srcStat, dstName, out)
	default:
{'fraction_non_alphanumeric': 0.14035639412997902, 'fraction_numerical': 0.004612159329140462, 'mean_word_length': 3.5541766109785202, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2670416', 'n_tokens_mistral': 3652, 'n_tokens_neox': 3425, 'n_words': 897}