<commit_before>package rpcd

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"github.com/Symantec/Dominator/lib/triggers"
	"github.com/Symantec/Dominator/proto/sub"
	"io"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"
	"time"
)

var (
	readOnly = flag.Bool("readOnly", false,
		"If true, refuse all Fetch and Update requests. For debugging only")
	disableUpdates = flag.Bool("disableUpdates", false,
		"If true, refuse all Update requests. For debugging only")
	disableTriggers = flag.Bool("disableTriggers", false,
		"If true, do not run any triggers. For debugging only")
)

var lastUpdateHadTriggerFailures bool

func (t *rpcType) Update(request sub.UpdateRequest,
	reply *sub.UpdateResponse) error {
	if *readOnly || *disableUpdates {
		txt := "Update() rejected due to read-only mode"
		logger.Println(txt)
		return errors.New(txt)
	}
	rwLock.Lock()
	defer rwLock.Unlock()
	fs := fileSystemHistory.FileSystem()
	if fs == nil {
		return errors.New("No file-system history yet")
	}
	logger.Printf("Update()\n")
	if fetchInProgress {
		logger.Println("Error: fetch already in progress")
		return errors.New("fetch already in progress")
	}
	if updateInProgress {
		logger.Println("Error: update progress")
		return errors.New("update in progress")
	}
	updateInProgress = true
	go doUpdate(request, fs.RootDirectoryName())
	return nil
}

func doUpdate(request sub.UpdateRequest, rootDirectoryName string) {
	defer clearUpdateInProgress()
	disableScannerFunc(true)
	defer disableScannerFunc(false)
	startTime := time.Now()
	var oldTriggers triggers.Triggers
	file, err := os.Open(oldTriggersFilename)
	if err == nil {
		decoder := json.NewDecoder(file)
		var trig triggers.Triggers
		err = decoder.Decode(&trig.Triggers)
		file.Close()
		if err == nil {
			oldTriggers = trig
		} else {
			logger.Printf("Error decoding old triggers: %s", err.Error())
		}
	}
	copyFilesToCache(request.FilesToCopyToCache, rootDirectoryName)
	makeObjectCopies(request.MultiplyUsedObjects)
	lastUpdateHadTriggerFailures = false
	if len(oldTriggers.Triggers) > 0 {
		makeInodes(request.InodesToMake, rootDirectoryName,
			request.MultiplyUsedObjects, &oldTriggers, false)
		makeHardlinks(request.HardlinksToMake, rootDirectoryName,
			&oldTriggers, false)
		doDeletes(request.PathsToDelete, rootDirectoryName, &oldTriggers, false)
		makeDirectories(request.DirectoriesToMake, rootDirectoryName,
			&oldTriggers, false)
		changeInodes(request.InodesToChange, rootDirectoryName, &oldTriggers,
			false)
		matchedOldTriggers := oldTriggers.GetMatchedTriggers()
		if runTriggers(matchedOldTriggers, "stop") {
			lastUpdateHadTriggerFailures = true
		}
	}
	makeInodes(request.InodesToMake, rootDirectoryName,
		request.MultiplyUsedObjects, request.Triggers, true)
	makeHardlinks(request.HardlinksToMake, rootDirectoryName,
		request.Triggers, true)
	doDeletes(request.PathsToDelete, rootDirectoryName, request.Triggers, true)
	makeDirectories(request.DirectoriesToMake, rootDirectoryName,
		request.Triggers, true)
	changeInodes(request.InodesToChange, rootDirectoryName, request.Triggers,
		true)
	matchedNewTriggers := request.Triggers.GetMatchedTriggers()
	file, err = os.Create(oldTriggersFilename)
	if err == nil {
		b, err := json.Marshal(request.Triggers.Triggers)
		if err == nil {
			var out bytes.Buffer
			json.Indent(&out, b, "", "    ")
			out.WriteTo(file)
		} else {
			logger.Printf("Error marshaling triggers: %s", err.Error())
		}
		file.Close()
	}
	if runTriggers(matchedNewTriggers, "start") {
		lastUpdateHadTriggerFailures = true
	}
	timeTaken := time.Since(startTime)
	logger.Printf("Update() completed in %s\n", timeTaken)
	// TODO(rgooch): Remove debugging hack and implement.
	time.Sleep(time.Second * 15)
	logger.Printf("Post-Update() debugging sleep complete\n")
}

func clearUpdateInProgress() {
	rwLock.Lock()
	defer rwLock.Unlock()
	updateInProgress = false
}

func copyFilesToCache(filesToCopyToCache []sub.FileToCopyToCache,
	rootDirectoryName string) {
	for _, fileToCopy := range filesToCopyToCache {
		sourcePathname := path.Join(rootDirectoryName, fileToCopy.Name)
		destPathname := path.Join(objectsDir,
			objectcache.HashToFilename(fileToCopy.Hash))<commit_msg>Remove 15 second debugging delay after Update() completed.<commit_after>package rpcd

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"github.com/Symantec/Dominator/lib/triggers"
	"github.com/Symantec/Dominator/proto/sub"
	"io"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"
	"time"
)

var (
	readOnly = flag.Bool("readOnly", false,
		"If true, refuse all Fetch and Update requests. For debugging only")
	disableUpdates = flag.Bool("disableUpdates", false,
		"If true, refuse all Update requests. For debugging only")
	disableTriggers = flag.Bool("disableTriggers", false,
		"If true, do not run any triggers. For debugging only")
)

var lastUpdateHadTriggerFailures bool

func (t *rpcType) Update(request sub.UpdateRequest,
	reply *sub.UpdateResponse) error {
	if *readOnly || *disableUpdates {
		txt := "Update() rejected due to read-only mode"
		logger.Println(txt)
		return errors.New(txt)
	}
	rwLock.Lock()
	defer rwLock.Unlock()
	fs := fileSystemHistory.FileSystem()
	if fs == nil {
		return errors.New("No file-system history yet")
	}
	logger.Printf("Update()\n")
	if fetchInProgress {
		logger.Println("Error: fetch already in progress")
		return errors.New("fetch already in progress")
	}
	if updateInProgress {
		logger.Println("Error: update progress")
		return errors.New("update in progress")
	}
	updateInProgress = true
	go doUpdate(request, fs.RootDirectoryName())
	return nil
}

func doUpdate(request sub.UpdateRequest, rootDirectoryName string) {
	defer clearUpdateInProgress()
	disableScannerFunc(true)
	defer disableScannerFunc(false)
	startTime := time.Now()
	var oldTriggers triggers.Triggers
	file, err := os.Open(oldTriggersFilename)
	if err == nil {
		decoder := json.NewDecoder(file)
		var trig triggers.Triggers
		err = decoder.Decode(&trig.Triggers)
		file.Close()
		if err == nil {
			oldTriggers = trig
		} else {
			logger.Printf("Error decoding old triggers: %s", err.Error())
		}
	}
	copyFilesToCache(request.FilesToCopyToCache, rootDirectoryName)
	makeObjectCopies(request.MultiplyUsedObjects)
	lastUpdateHadTriggerFailures = false
	if len(oldTriggers.Triggers) > 0 {
		makeInodes(request.InodesToMake, rootDirectoryName,
			request.MultiplyUsedObjects, &oldTriggers, false)
		makeHardlinks(request.HardlinksToMake, rootDirectoryName,
			&oldTriggers, false)
		doDeletes(request.PathsToDelete, rootDirectoryName, &oldTriggers, false)
		makeDirectories(request.DirectoriesToMake, rootDirectoryName,
			&oldTriggers, false)
		changeInodes(request.InodesToChange, rootDirectoryName, &oldTriggers,
			false)
		matchedOldTriggers := oldTriggers.GetMatchedTriggers()
		if runTriggers(matchedOldTriggers, "stop") {
			lastUpdateHadTriggerFailures = true
		}
	}
	makeInodes(request.InodesToMake, rootDirectoryName,
		request.MultiplyUsedObjects, request.Triggers, true)
	makeHardlinks(request.HardlinksToMake, rootDirectoryName,
		request.Triggers, true)
	doDeletes(request.PathsToDelete, rootDirectoryName, request.Triggers, true)
	makeDirectories(request.DirectoriesToMake, rootDirectoryName,
		request.Triggers, true)
	changeInodes(request.InodesToChange, rootDirectoryName, request.Triggers,
		true)
	matchedNewTriggers := request.Triggers.GetMatchedTriggers()
	file, err = os.Create(oldTriggersFilename)
	if err == nil {
		b, err := json.Marshal(request.Triggers.Triggers)
		if err == nil {
			var out bytes.Buffer
			json.Indent(&out, b, "", "    ")
			out.WriteTo(file)
		} else {
			logger.Printf("Error marshaling triggers: %s", err.Error())
		}
		file.Close()
	}
	if runTriggers(matchedNewTriggers, "start") {
		lastUpdateHadTriggerFailures = true
	}
	timeTaken := time.Since(startTime)
	logger.Printf("Update() completed in %s\n", timeTaken)
}

func clearUpdateInProgress() {
	rwLock.Lock()
	defer rwLock.Unlock()
	updateInProgress = false
}

func copyFilesToCache(filesToCopyToCache []sub.FileToCopyToCache,
	rootDirectoryName string) {
	for _, fileToCopy := range filesToCopyToCache {
		sourcePathname := path.Join(rootDirectoryName, fileToCopy.Name)
		destPathname := path.Join(objectsDir,
			objectcache.HashToFilename(fileToCopy.Hash))
{'fraction_non_alphanumeric': 0.11443600233781415, 'fraction_numerical': 0.0007013442431326709, 'mean_word_length': 5.413793103448276, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4849210', 'n_tokens_mistral': 3004, 'n_tokens_neox': 2896, 'n_words': 588}