<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/filter"
	"github.com/Symantec/Dominator/lib/fsbench"
	"github.com/Symantec/Dominator/lib/fsrateio"
	"github.com/Symantec/Dominator/lib/html"
	"github.com/Symantec/Dominator/lib/logbuf"
	"github.com/Symantec/Dominator/lib/memstats"
	"github.com/Symantec/Dominator/lib/rateio"
	"github.com/Symantec/Dominator/sub/httpd"
	"github.com/Symantec/Dominator/sub/rpcd"
	"github.com/Symantec/Dominator/sub/scanner"
	"github.com/Symantec/tricorder/go/tricorder"
	"io"
	"log"
	"os"
	"os/signal"
	"path"
	"runtime"
	"strconv"
	"syscall"
)

var (
	caFile = flag.String("CAfile", "/etc/ssl/CA.pem",
		"Name of file containing the root of trust")
	certFile = flag.String("certFile", "/etc/ssl/subd/cert.pem",
		"Name of file containing the SSL certificate")
	keyFile = flag.String("keyFile", "/etc/ssl/subd/key.pem",
		"Name of file containing the SSL key")
	logbufLines = flag.Uint("logbufLines", 1024,
		"Number of lines to store in the log buffer")
	maxThreads = flag.Uint("maxThreads", 1,
		"Maximum number of parallel OS threads to use")
	pidfile = flag.String("pidfile", "/var/run/subd.pid",
		"Name of file to write my PID to")
	portNum = flag.Uint("portNum", constants.SubPortNumber,
		"Port number to allocate and listen on for HTTP/RPC")
	rootDir = flag.String("rootDir", "/",
		"Name of root of directory tree to manage")
	showStats = flag.Bool("showStats", false,
		"If true, show statistics after each cycle")
	subdDir = flag.String("subdDir", ".subd",
		"Name of subd private directory, relative to rootDir. This must be on the same file-system as rootDir")
	unshare = flag.Bool("unshare", true, "Internal use only.")
)

func sanityCheck() bool {
	r_devnum, err := fsbench.GetDevnumForFile(*rootDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to get device number for: %s\t%s\n",
			*rootDir, err)
		return false
	}
	subdDirPathname := path.Join(*rootDir, *subdDir)
	s_devnum, err := fsbench.GetDevnumForFile(subdDirPathname)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to get device number for: %s\t%s\n",
			subdDirPathname, err)
		return false
	}
	if r_devnum != s_devnum {
		fmt.Fprintf(os.Stderr,
			"rootDir and subdDir must be on the same file-system\n")
		return false
	}
	return true
}

func createDirectory(dirname string) bool {
	if err := os.MkdirAll(dirname, 0750); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to create directory: %s\t%s\n",
			dirname, err)
		return false
	}
	return true
}

func mountTmpfs(dirname string) bool {
	var statfs syscall.Statfs_t
	if err := syscall.Statfs(dirname, &statfs); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to create Statfs: %s\t%s\n",
			dirname, err)
		return false
	}
	if statfs.Type != 0x01021994 {
		err := syscall.Mount("none", dirname, "tmpfs", 0,
			"size=65536,mode=0750")
		if err == nil {
			fmt.Printf("Mounted tmpfs on: %s\n", dirname)
		} else {
			fmt.Fprintf(os.Stderr, "Unable to mount tmpfs on: %s\t%s\n",
				dirname, err)
			return false
		}
	}
	return true
}

func unshareAndBind(workingRootDir string) bool {
	if *unshare {
		// Re-exec myself using the unshare syscall while on a locked thread.
		// This hack is required because syscall.Unshare() operates on only one
		// thread in the process, and Go switches execution between threads
		// randomly. Thus, the namespace can be suddenly switched for running
		// code. This is an aspect of Go that was not well thought out.
		runtime.LockOSThread()
		if err := syscall.Unshare(syscall.CLONE_NEWNS); err != nil {
			fmt.Fprintf(os.Stderr, "Unable to unshare mount namesace\t%s\n",
				err)
			return false
		}
		args := append(os.Args, "-unshare=false")
		if err := syscall.Exec(args[0], args, os.Environ()); err != nil {
			fmt.Fprintf(os.Stderr, "Unable to Exec:%s\t%s\n", args[0], err)
			return false
		}
	}
	err := syscall.Mount("none", "/", "", syscall.MS_REC|syscall.MS_PRIVATE, "")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to set mount sharing to private\t%s\n",
			err)
		return false
	}
	syscall.Unmount(workingRootDir, 0)
	err = syscall.Mount(*rootDir, workingRootDir, "", syscall.MS_BIND, "")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to bind mount %s to %s\t%s\n",
			*rootDir, workingRootDir, err)
		return false
	}
	return true
}

func getCachedFsSpeed(workingRootDir string, cacheDirname string) (bytesPerSecond,
	blocksPerSecond uint64, computed, ok bool) {
	bytesPerSecond = 0
	blocksPerSecond = 0
	devnum, err := fsbench.GetDevnumForFile(workingRootDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to get device number for: %s\t%s\n",<commit_msg>Change subd so that the whole process is at least slightly niced.<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/filter"
	"github.com/Symantec/Dominator/lib/fsbench"
	"github.com/Symantec/Dominator/lib/fsrateio"
	"github.com/Symantec/Dominator/lib/html"
	"github.com/Symantec/Dominator/lib/logbuf"
	"github.com/Symantec/Dominator/lib/memstats"
	"github.com/Symantec/Dominator/lib/rateio"
	"github.com/Symantec/Dominator/sub/httpd"
	"github.com/Symantec/Dominator/sub/rpcd"
	"github.com/Symantec/Dominator/sub/scanner"
	"github.com/Symantec/tricorder/go/tricorder"
	"io"
	"log"
	"os"
	"os/signal"
	"path"
	"runtime"
	"strconv"
	"syscall"
)

var (
	caFile = flag.String("CAfile", "/etc/ssl/CA.pem",
		"Name of file containing the root of trust")
	certFile = flag.String("certFile", "/etc/ssl/subd/cert.pem",
		"Name of file containing the SSL certificate")
	keyFile = flag.String("keyFile", "/etc/ssl/subd/key.pem",
		"Name of file containing the SSL key")
	logbufLines = flag.Uint("logbufLines", 1024,
		"Number of lines to store in the log buffer")
	maxThreads = flag.Uint("maxThreads", 1,
		"Maximum number of parallel OS threads to use")
	pidfile = flag.String("pidfile", "/var/run/subd.pid",
		"Name of file to write my PID to")
	portNum = flag.Uint("portNum", constants.SubPortNumber,
		"Port number to allocate and listen on for HTTP/RPC")
	rootDir = flag.String("rootDir", "/",
		"Name of root of directory tree to manage")
	showStats = flag.Bool("showStats", false,
		"If true, show statistics after each cycle")
	subdDir = flag.String("subdDir", ".subd",
		"Name of subd private directory, relative to rootDir. This must be on the same file-system as rootDir")
	unshare = flag.Bool("unshare", true, "Internal use only.")
)

func sanityCheck() bool {
	r_devnum, err := fsbench.GetDevnumForFile(*rootDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to get device number for: %s\t%s\n",
			*rootDir, err)
		return false
	}
	subdDirPathname := path.Join(*rootDir, *subdDir)
	s_devnum, err := fsbench.GetDevnumForFile(subdDirPathname)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to get device number for: %s\t%s\n",
			subdDirPathname, err)
		return false
	}
	if r_devnum != s_devnum {
		fmt.Fprintf(os.Stderr,
			"rootDir and subdDir must be on the same file-system\n")
		return false
	}
	return true
}

func createDirectory(dirname string) bool {
	if err := os.MkdirAll(dirname, 0750); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to create directory: %s\t%s\n",
			dirname, err)
		return false
	}
	return true
}

func mountTmpfs(dirname string) bool {
	var statfs syscall.Statfs_t
	if err := syscall.Statfs(dirname, &statfs); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to create Statfs: %s\t%s\n",
			dirname, err)
		return false
	}
	if statfs.Type != 0x01021994 {
		err := syscall.Mount("none", dirname, "tmpfs", 0,
			"size=65536,mode=0750")
		if err == nil {
			fmt.Printf("Mounted tmpfs on: %s\n", dirname)
		} else {
			fmt.Fprintf(os.Stderr, "Unable to mount tmpfs on: %s\t%s\n",
				dirname, err)
			return false
		}
	}
	return true
}

func unshareAndBind(workingRootDir string) bool {
	if *unshare {
		// Re-exec myself using the unshare syscall while on a locked thread.
		// This hack is required because syscall.Unshare() operates on only one
		// thread in the process, and Go switches execution between threads
		// randomly. Thus, the namespace can be suddenly switched for running
		// code. This is an aspect of Go that was not well thought out.
		runtime.LockOSThread()
		if err := syscall.Unshare(syscall.CLONE_NEWNS); err != nil {
			fmt.Fprintf(os.Stderr, "Unable to unshare mount namesace\t%s\n",
				err)
			return false
		}
		// Ensure the process is slightly niced. Since the Linux implementation
		// of setpriority(2) only applies to a thread, not the whole process
		// (contrary to the POSIX specification), do this in the pinned OS
		// thread so that the whole process (after exec) will be niced.
		syscall.Setpriority(syscall.PRIO_PROCESS, 0, 1)
		args := append(os.Args, "-unshare=false")
		if err := syscall.Exec(args[0], args, os.Environ()); err != nil {
			fmt.Fprintf(os.Stderr, "Unable to Exec:%s\t%s\n", args[0], err)
			return false
		}
	}
	err := syscall.Mount("none", "/", "", syscall.MS_REC|syscall.MS_PRIVATE, "")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to set mount sharing to private\t%s\n",
			err)
		return false
	}
	syscall.Unmount(workingRootDir, 0)
	err = syscall.Mount(*rootDir, workingRootDir, "", syscall.MS_BIND, "")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to bind mount %s to %s\t%s\n",
			*rootDir, workingRootDir, err)
		return false
	}
	return true
}

func getCachedFsSpeed(workingRootDir string, cacheDirname string) (bytesPerSecond,
	blocksPerSecond uint64, computed, ok bool) {
	bytesPerSecond = 0
	blocksPerSecond = 0
	devnum, err := fsbench.GetDevnumForFile(workingRootDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to get device number for: %s\t%s\n",
{'fraction_non_alphanumeric': 0.13608311047109647, 'fraction_numerical': 0.00750874305698416, 'mean_word_length': 4.0352149145520455, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '60043', 'n_tokens_mistral': 3807, 'n_tokens_neox': 3542, 'n_words': 950}