<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
	"text/template"
)

const (
	hangMessage      = "The following program hangs with the provided go-fuzz output:"
	panicMessage     = "The following program panics with the provided output:"
	standardTemplate = `{{ .Message }}

` + "```" + `
{{ .Program }}
` + "```" + `

-----

Standard output:

` + "```" + `
{{ .Output }}
` + "```" + `

Output ran through [panicparse](https://github.com/maruel/panicparse):


` + "```" + `
{{ .PanicParse }}
` + "```" + `


This {{ .Type }} was discovered with [go-fuzz](https://github.com/dvyukov/go-fuzz).
`
)

type CrashDescription struct {
	Type, Message, Program, Output, PanicParse string
}

type Application struct {
	QuotedData string
}

func main() {
	if len(os.Args) != 3 {
		fmt.Fprintf(os.Stderr, "Invalid arguments given. Usage: gfig <applicationTemplate> <unquotedCrashFile>")

		return
	}

	appTemplateText, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}

	applicationTemplate := template.Must(template.New("application").Parse(string(appTemplateText)))
	descriptionTemplate := template.Must(template.New("application").Parse(standardTemplate))

	var (
		applicationOutput bytes.Buffer
		descriptionOutput bytes.Buffer
	)

	quotedData, err := ioutil.ReadFile(os.Args[2] + ".quoted")
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}

	// Get the application sample
	if err := applicationTemplate.Execute(&applicationOutput, Application{
		string(quotedData),
	}); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}

	// Read go-fuzz's output
	output, err := ioutil.ReadFile(os.Args[2] + ".output")
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}

	message := panicMessage
	crashType := "panic"

	if strings.HasPrefix(string(output), "program hanged") {
		message = hangMessage
		crashType = "hang"
	} else {
		file, err := os.Create(path.Join(os.TempDir(), "gfig_test.go"))
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(-1)
		}

		outputCopy := bytes.NewBuffer(applicationOutput.Bytes())
		io.Copy(file, outputCopy)
		file.Close()

		goRunCommand := exec.Command("go", "run", file.Name())

		output, _ = goRunCommand.CombinedOutput()
	}

	if err := descriptionTemplate.Execute(&descriptionOutput, NewCrashDescription(crashType, message, string(applicationOutput.Bytes()), string(output))); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}

	ioutil.WriteFile(os.Args[2]+"_description.md", descriptionOutput.Bytes(), os.ModePerm)
}

func NewCrashDescription(crashType, message, program, output string) CrashDescription {
	goRunCommand := exec.Command("panicparse")
	goRunCommand.Stdin = strings.NewReader(output)

	goRunStdout, err := goRunCommand.Output()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}

	return CrashDescription{
		Type:       crashType,
		Message:    message,
		Program:    program,
		PanicParse: string(goRunStdout),
		Output:     output,
	}
}<commit_msg>Improve output to identify non-crashers and add git info<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
	"text/template"
)

const (
	hangMessage      = "hangs with the provided go-fuzz output:"
	panicMessage     = "panics with the provided output:"
	standardTemplate = `The following program at revision ` + "`" + `{{ .Revision }}` + "`" + ` {{ .Message }}

` + "```" + `
{{ .Program }}
` + "```" + `

-----

Standard output:

` + "```" + `
{{ .Output }}
` + "```" + `

Output ran through [panicparse](https://github.com/maruel/panicparse):


` + "```" + `
{{ .PanicParse }}
` + "```" + `


This {{ .Type }} was discovered with [go-fuzz](https://github.com/dvyukov/go-fuzz).
`
)

type CrashDescription struct {
	Type, Message, Program, Output, PanicParse, Revision string
}

type Application struct {
	QuotedData string
}

func main() {
	if len(os.Args) != 4 {
		fmt.Fprintf(os.Stderr, "Invalid arguments given. Usage: gfig <gitRepo> <applicationTemplate> <unquotedCrashFile>")

		return
	}

	gitRepo := os.Args[1]
	appTemplatePath := os.Args[2]
	crashFile := os.Args[3]

	appTemplateText, err := ioutil.ReadFile(appTemplatePath)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}

	applicationTemplate := template.Must(template.New("application").Parse(string(appTemplateText)))
	descriptionTemplate := template.Must(template.New("application").Parse(standardTemplate))

	var (
		applicationOutput bytes.Buffer
		descriptionOutput bytes.Buffer
	)

	quotedData, err := ioutil.ReadFile(crashFile + ".quoted")
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}

	// Get the application sample
	if err := applicationTemplate.Execute(&applicationOutput, Application{
		string(quotedData),
	}); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}

	// Read go-fuzz's output
	output, err := ioutil.ReadFile(crashFile + ".output")
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}

	message := panicMessage
	crashType := "panic"

	if strings.HasPrefix(string(output), "program hanged") {
		message = hangMessage
		crashType = "hang"
	} else {
		file, err := os.Create(path.Join(os.TempDir(), "gfig_sample.go"))
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(-1)
		}

		outputCopy := bytes.NewBuffer(applicationOutput.Bytes())
		io.Copy(file, outputCopy)
		file.Close()

		goRunCommand := exec.Command("go", "run", file.Name())

		output, _ = goRunCommand.CombinedOutput()

		if len(output) == 0 {
			fmt.Fprintln(os.Stderr, crashFile, "is not a valid crash")
			os.Exit(0)
		}
	}

	crashDescription := NewCrashDescription(gitRepo, crashType, message, string(applicationOutput.Bytes()), string(output))

	if err := descriptionTemplate.Execute(&descriptionOutput, crashDescription); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}

	ioutil.WriteFile(crashFile+"_description.md", descriptionOutput.Bytes(), os.ModePerm)
}

func NewCrashDescription(gitRepo, crashType, message, program, output string) CrashDescription {
	goRunCommand := exec.Command("panicparse")
	goRunCommand.Stdin = strings.NewReader(output)

	goRunStdout, err := goRunCommand.Output()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}

	return CrashDescription{
		Type:       crashType,
		Message:    message,
		Program:    program,
		PanicParse: string(goRunStdout),
		Output:     output,
		Revision:   GitDescribeRepo(gitRepo),
	}
}

func GitDescribeRepo(repoPath string) string {
	gitDescribeCommand := exec.Command("git", "describe", "--all", "--long")
	gitDescribeCommand.Dir = repoPath

	gitStdout, err := gitDescribeCommand.Output()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}

	return string(gitStdout)
}
{'fraction_non_alphanumeric': 0.1624633431085044, 'fraction_numerical': 0.003812316715542522, 'mean_word_length': 4.105538922155689, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7509523', 'n_tokens_mistral': 2538, 'n_tokens_neox': 2403, 'n_words': 481}