<commit_before>package main

// Benchmark the scanning of a file-system tree.
// Usage: scan [dirname [numScans]]
//   dirname:   the top of the directory tree to scan (default=/)
//   numScans:  the number of scans to run (default=1, infinite: < 0)

import (
	"fmt"
	"github.com/Symantec/Dominator/sub/fsrateio"
	"github.com/Symantec/Dominator/sub/scanner"
	"os"
	"strconv"
	"syscall"
	"time"
)

func main() {
	pathname := "/"
	var numScans int = 1
	var err error
	if len(os.Args) >= 2 {
		pathname = os.Args[1]
	}
	if len(os.Args) == 3 {
		numScans, err = strconv.Atoi(os.Args[2])
		if err != nil {
			fmt.Printf("Error! %s\n", err)
			return
		}
	}
	ctx, err := fsrateio.NewContext(pathname)
	if err != nil {
		fmt.Printf("Error! %s\n", err)
		return
	}
	fmt.Println(ctx)
	syscall.Setpriority(syscall.PRIO_PROCESS, 0, 10)
	var prev_fs *scanner.FileSystem
	for iter := 1; numScans < 0 || iter <= numScans; iter++ {
		timeStart := time.Now()
		fs, err := scanner.ScanFileSystem(pathname, ctx)
		timeStop := time.Now()
		if err != nil {
			fmt.Printf("Error! %s\n", err)
			return
		}
		var tread uint64 = 0
		for _, inode := range fs.InodeTable {
			tread += inode.Length()
		}
		fmt.Printf("Total scanned: %s,\t", fsrateio.FormatBytes(tread))
		bytesPerSecond := uint64(float64(tread) /
			timeStop.Sub(timeStart).Seconds())
		fmt.Printf("%s/s\n", fsrateio.FormatBytes(bytesPerSecond))
		if prev_fs != nil {
			if !scanner.Compare(prev_fs, fs, nil) {
				fmt.Println("Scan results different from last run")
			}
		}
		prev_fs = fs
	}<commit_msg>Switch scan utility back to os.Stdout for compare logging.<commit_after>package main

// Benchmark the scanning of a file-system tree.
// Usage: scan [dirname [numScans]]
//   dirname:   the top of the directory tree to scan (default=/)
//   numScans:  the number of scans to run (default=1, infinite: < 0)

import (
	"fmt"
	"github.com/Symantec/Dominator/sub/fsrateio"
	"github.com/Symantec/Dominator/sub/scanner"
	"os"
	"strconv"
	"syscall"
	"time"
)

func main() {
	pathname := "/"
	var numScans int = 1
	var err error
	if len(os.Args) >= 2 {
		pathname = os.Args[1]
	}
	if len(os.Args) == 3 {
		numScans, err = strconv.Atoi(os.Args[2])
		if err != nil {
			fmt.Printf("Error! %s\n", err)
			return
		}
	}
	ctx, err := fsrateio.NewContext(pathname)
	if err != nil {
		fmt.Printf("Error! %s\n", err)
		return
	}
	fmt.Println(ctx)
	syscall.Setpriority(syscall.PRIO_PROCESS, 0, 10)
	var prev_fs *scanner.FileSystem
	for iter := 1; numScans < 0 || iter <= numScans; iter++ {
		timeStart := time.Now()
		fs, err := scanner.ScanFileSystem(pathname, ctx)
		timeStop := time.Now()
		if err != nil {
			fmt.Printf("Error! %s\n", err)
			return
		}
		var tread uint64 = 0
		for _, inode := range fs.InodeTable {
			tread += inode.Length()
		}
		fmt.Printf("Total scanned: %s,\t", fsrateio.FormatBytes(tread))
		bytesPerSecond := uint64(float64(tread) /
			timeStop.Sub(timeStart).Seconds())
		fmt.Printf("%s/s\n", fsrateio.FormatBytes(bytesPerSecond))
		if prev_fs != nil {
			if !scanner.Compare(prev_fs, fs, os.Stdout) {
				fmt.Println("Scan results different from last run")
			}
		}
		prev_fs = fs
	}
{'fraction_non_alphanumeric': 0.15675675675675677, 'fraction_numerical': 0.012082670906200318, 'mean_word_length': 3.4309859154929576, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '866745', 'n_tokens_mistral': 1347, 'n_tokens_neox': 1227, 'n_words': 294}