<commit_before>// Copyright 2016 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// syz-crush replays crash log on multiple VMs. Usage:
//   syz-crush -config=config.file execution.log
// Intended for reproduction of particularly elusive crashes.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"time"

	"github.com/google/syzkaller/pkg/csource"
	"github.com/google/syzkaller/pkg/hash"
	"github.com/google/syzkaller/pkg/instance"
	"github.com/google/syzkaller/pkg/mgrconfig"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/pkg/report"
	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/vm"
)

var (
	flagConfig      = flag.String("config", "", "manager configuration file")
	flagDebug       = flag.Bool("debug", false, "dump all VM output to console")
	flagRestartTime = flag.Duration("restart_time", time.Hour, "how long to run the test")
	flagInfinite    = flag.Bool("infinite", true, "by default test is run for ever, -infinite=false to stop on crash")
)

type FileType int

const (
	LogFile FileType = iota
	CProg
)

func main() {
	flag.Parse()
	if len(flag.Args()) != 1 || *flagConfig == "" {
		fmt.Fprintf(os.Stderr, "usage: syz-crush [flags] <execution.log|creprog.c>\n")
		flag.PrintDefaults()
		os.Exit(1)
	}
	cfg, err := mgrconfig.LoadFile(*flagConfig)
	if err != nil {
		log.Fatal(err)
	}

	if *flagInfinite {
		log.Printf("running infinitely and restarting VM every %v", *flagRestartTime)
	} else {
		log.Printf("running until crash is found or till %v", *flagRestartTime)
	}

	vmPool, err := vm.Create(cfg, *flagDebug)
	if err != nil {
		log.Fatalf("%v", err)
	}

	reporter, err := report.NewReporter(cfg)
	if err != nil {
		log.Fatalf("%v", err)
	}

	reproduceMe := flag.Args()[0]
	if cfg.Tag == "" {
		// If no tag is given, use reproducer name as the tag.
		cfg.Tag = filepath.Base(reproduceMe)
	}
	runType := LogFile
	if strings.HasSuffix(reproduceMe, ".c") {
		runType = CProg
	}
	if runType == CProg {
		execprog, err := ioutil.ReadFile(reproduceMe)
		if err != nil {
			log.Fatalf("error reading source file from '%s'", reproduceMe)
		}

		cfg.ExecprogBin, err = csource.BuildNoWarn(cfg.Target, execprog)
		if err != nil {
			log.Fatalf("failed to build source file: %v", err)
		}

		log.Printf("compiled csource %v to cprog: %v", reproduceMe, cfg.ExecprogBin)
	} else {
		log.Printf("reproducing from log file: %v", reproduceMe)
	}

	log.Printf("booting %v test machines...", vmPool.Count())
	runDone := make(chan *report.Report)
	var shutdown, stoppedWorkers uint32

	for i := 0; i < vmPool.Count(); i++ {
		go func(index int) {
			for {
				runDone <- runInstance(cfg.Target, cfg, reporter, vmPool, index, *flagRestartTime, runType)
				if atomic.LoadUint32(&shutdown) != 0 || !*flagInfinite {
					// If this is the last worker then we can close the channel.
					if atomic.AddUint32(&stoppedWorkers, 1) == uint32(vmPool.Count()) {
						log.Printf("vm-%v: closing channel", index)
						close(runDone)
					}
					break
				}
			}
			log.Printf("vm-%v: done", index)
		}(i)
	}

	shutdownC := make(chan struct{})
	osutil.HandleInterrupts(shutdownC)
	go func() {
		<-shutdownC
		atomic.StoreUint32(&shutdown, 1)
		close(vm.Shutdown)
	}()

	var count, crashes int
	for rep := range runDone {
		count++
		if rep != nil {
			crashes++
			storeCrash(cfg, rep)
		}
		log.Printf("instances executed: %v, crashes: %v", count, crashes)
	}

	log.Printf("all done. reproduced %v crashes. reproduce rate %.2f%%", crashes, float64(crashes)/float64(count)*100.0)
}

func storeCrash(cfg *mgrconfig.Config, rep *report.Report) {
	id := hash.String([]byte(rep.Title))
	dir := filepath.Join(cfg.Workdir, "crashes", id)
	osutil.MkdirAll(dir)
<commit_msg>tools/syz-crush: make the results saving more flexible<commit_after>// Copyright 2016 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// syz-crush replays crash log on multiple VMs. Usage:
//   syz-crush -config=config.file execution.log
// Intended for reproduction of particularly elusive crashes.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"time"

	"github.com/google/syzkaller/pkg/csource"
	"github.com/google/syzkaller/pkg/hash"
	"github.com/google/syzkaller/pkg/instance"
	"github.com/google/syzkaller/pkg/mgrconfig"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/pkg/report"
	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/vm"
)

var (
	flagConfig      = flag.String("config", "", "manager configuration file")
	flagDebug       = flag.Bool("debug", false, "dump all VM output to console")
	flagRestartTime = flag.Duration("restart_time", time.Hour, "how long to run the test")
	flagInfinite    = flag.Bool("infinite", true, "by default test is run for ever, -infinite=false to stop on crash")
)

type FileType int

const (
	LogFile FileType = iota
	CProg
)

func main() {
	flag.Parse()
	if len(flag.Args()) != 1 || *flagConfig == "" {
		fmt.Fprintf(os.Stderr, "usage: syz-crush [flags] <execution.log|creprog.c>\n")
		flag.PrintDefaults()
		os.Exit(1)
	}
	cfg, err := mgrconfig.LoadFile(*flagConfig)
	if err != nil {
		log.Fatal(err)
	}

	if *flagInfinite {
		log.Printf("running infinitely and restarting VM every %v", *flagRestartTime)
	} else {
		log.Printf("running until crash is found or till %v", *flagRestartTime)
	}

	vmPool, err := vm.Create(cfg, *flagDebug)
	if err != nil {
		log.Fatalf("%v", err)
	}

	reporter, err := report.NewReporter(cfg)
	if err != nil {
		log.Fatalf("%v", err)
	}

	reproduceMe := flag.Args()[0]
	if cfg.Tag == "" {
		// If no tag is given, use reproducer name as the tag.
		cfg.Tag = filepath.Base(reproduceMe)
	}
	runType := LogFile
	if strings.HasSuffix(reproduceMe, ".c") {
		runType = CProg
	}
	if runType == CProg {
		execprog, err := ioutil.ReadFile(reproduceMe)
		if err != nil {
			log.Fatalf("error reading source file from '%s'", reproduceMe)
		}

		cfg.ExecprogBin, err = csource.BuildNoWarn(cfg.Target, execprog)
		if err != nil {
			log.Fatalf("failed to build source file: %v", err)
		}

		log.Printf("compiled csource %v to cprog: %v", reproduceMe, cfg.ExecprogBin)
	} else {
		log.Printf("reproducing from log file: %v", reproduceMe)
	}

	log.Printf("booting %v test machines...", vmPool.Count())
	runDone := make(chan *report.Report)
	var shutdown, stoppedWorkers uint32

	for i := 0; i < vmPool.Count(); i++ {
		go func(index int) {
			for {
				runDone <- runInstance(cfg.Target, cfg, reporter, vmPool, index, *flagRestartTime, runType)
				if atomic.LoadUint32(&shutdown) != 0 || !*flagInfinite {
					// If this is the last worker then we can close the channel.
					if atomic.AddUint32(&stoppedWorkers, 1) == uint32(vmPool.Count()) {
						log.Printf("vm-%v: closing channel", index)
						close(runDone)
					}
					break
				}
			}
			log.Printf("vm-%v: done", index)
		}(i)
	}

	shutdownC := make(chan struct{})
	osutil.HandleInterrupts(shutdownC)
	go func() {
		<-shutdownC
		atomic.StoreUint32(&shutdown, 1)
		close(vm.Shutdown)
	}()

	var count, crashes int
	for rep := range runDone {
		count++
		if rep != nil {
			crashes++
			storeCrash(cfg, rep)
		}
		log.Printf("instances executed: %v, crashes: %v", count, crashes)
	}

	log.Printf("all done. reproduced %v crashes. reproduce rate %.2f%%", crashes, float64(crashes)/float64(count)*100.0)
}

func storeCrash(cfg *mgrconfig.Config, rep *report.Report) {
	id := hash.String([]byte(rep.Title))
	dir := filepath.Join(filepath.Dir(flag.Args()[0]), "crashes", id)
	osutil.MkdirAll(dir)

{'fraction_non_alphanumeric': 0.14925373134328357, 'fraction_numerical': 0.008106021616057642, 'mean_word_length': 3.7922318125770653, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4117842', 'n_tokens_mistral': 3066, 'n_tokens_neox': 2855, 'n_words': 714}