<commit_before>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// Package csource generates [almost] equivalent C programs from syzkaller programs.
//
// Outline of the process:
// - inputs to the generation are the program and options
// - options control multiple aspects of the resulting C program,
//   like if we want a multi-threaded program or a single-threaded,
//   what type of sandbox we want to use, if we want to setup net devices or not, etc
// - we use actual executor sources as the base
// - gen.go takes all executor/common*.h headers and bundles them into generated.go
// - during generation we tear executor headers apart and take only the bits
//   we need for the current program/options, this is done by running C preprocessor
//   with particular set of defines so that the preprocessor removes unneeded
//   #ifdef SYZ_FOO sections
// - then we generate actual syscall calls with the given arguments
//   based on the binary "encodingexec" representation of the program
//   (the same representation executor uses for interpretation)
// - then we glue it all together
// - as the last step we run some text post-processing on the resulting source code:
//   remove debug calls, replace exitf/fail with exit, hoist/sort/dedup includes,
//   remove duplicate empty lines, etc
package csource

import (
	"bytes"
	"fmt"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/sys/targets"
)

// Write generates C source for program p based on the provided options opt.
func Write(p *prog.Prog, opts Options) ([]byte, error) {
	if err := opts.Check(p.Target.OS); err != nil {
		return nil, fmt.Errorf("csource: invalid opts: %v", err)
	}
	ctx := &context{
		p:         p,
		opts:      opts,
		target:    p.Target,
		sysTarget: targets.Get(p.Target.OS, p.Target.Arch),
		calls:     make(map[string]uint64),
	}

	ctx.filterCalls()
	calls, vars, err := ctx.generateProgCalls(ctx.p, opts.Trace)
	if err != nil {
		return nil, err
	}

	mmapProg := p.Target.DataMmapProg()
	mmapCalls, _, err := ctx.generateProgCalls(mmapProg, false)
	if err != nil {
		return nil, err
	}

	for _, c := range append(mmapProg.Calls, p.Calls...) {
		ctx.calls[c.Meta.CallName] = c.Meta.NR
	}

	varsBuf := new(bytes.Buffer)
	if len(vars) != 0 {
		fmt.Fprintf(varsBuf, "uint64 r[%v] = {", len(vars))
		for i, v := range vars {
			if i != 0 {
				fmt.Fprintf(varsBuf, ", ")
			}
			fmt.Fprintf(varsBuf, "0x%x", v)
		}
		fmt.Fprintf(varsBuf, "};\n")
	}

	sandboxFunc := "loop();"
	if opts.Sandbox != "" {
		sandboxFunc = "do_sandbox_" + opts.Sandbox + "();"
	}
	replacements := map[string]string{
		"PROCS":           fmt.Sprint(opts.Procs),
		"REPEAT_TIMES":    fmt.Sprint(opts.RepeatTimes),
		"NUM_CALLS":       fmt.Sprint(len(p.Calls)),
		"MMAP_DATA":       strings.Join(mmapCalls, ""),
		"SYSCALL_DEFINES": ctx.generateSyscallDefines(),
		"SANDBOX_FUNC":    sandboxFunc,
		"RESULTS":         varsBuf.String(),
		"SYSCALLS":        ctx.generateSyscalls(calls, len(vars) != 0),
	}
	if !opts.Threaded && !opts.Repeat && opts.Sandbox == "" {
		// This inlines syscalls right into main for the simplest case.
		replacements["SANDBOX_FUNC"] = replacements["SYSCALLS"]
		replacements["SYSCALLS"] = "unused"
	}
	timeouts := ctx.sysTarget.Timeouts(opts.Slowdown)
	replacements["PROGRAM_TIMEOUT_MS"] = fmt.Sprint(int(timeouts.Program / time.Millisecond))
	timeoutExpr := fmt.Sprint(int(timeouts.Syscall / time.Millisecond))
	for i, call := range p.Calls {
		if timeout := call.Meta.Attrs.Timeout; timeout != 0 {
			timeoutExpr += fmt.Sprintf(" + (call == %v ? %v : 0)", i, timeout*uint64(timeouts.Scale))
		}
	}
	replacements["CALL_TIMEOUT_MS"] = timeoutExpr
	result, err := createCommonHeader(p, mmapProg, replacements, opts)
	if err != nil {
		return nil, err
	}
	const header = "// autogenerated by syzkaller (https://github.com/google/syzkaller)\n\n"
	result = append([]byte(header), result...)
	result = ctx.postProcess(result)
	return result, nil
}

type context struct {
	p         *prog.Prog
	opts      Options
	target    *prog.Target
	sysTarget *targets.Target
	calls     map[string]uint64 // CallName -> NR
}

// This is a kludge, but we keep it here until a better approach is implemented.
// TODO: untie syz_emit_ethernet/syz_extract_tcp_res and NetInjection. And also
// untie VhciInjection and syz_emit_vhci. Then we could remove this method.
func (ctx *context) filterCalls() {
	p := ctx.p
	for i := 0; i < len(p.Calls); {
		call := p.Calls[i]
		callName := call.Meta.CallName
		emitCall := (ctx.opts.NetInjection ||
			callName != "syz_emit_ethernet" &&
				callName != "syz_extract_tcp_res") &&
			(ctx.opts.VhciInjection || callName != "syz_emit_vhci")
		if emitCall {
			i++
			continue
		}
		// Remove the call.
		if ctx.p == p {
			// We lazily clone the program to avoid unnecessary copying.
			p = ctx.p.Clone()
		}
		p.RemoveCall(i)
	}
	ctx.p = p
}

func (ctx *context) generateSyscalls(calls []string, hasVars bool) string {<commit_msg>pkg/csource: fix call list filtering not being consistent<commit_after>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// Package csource generates [almost] equivalent C programs from syzkaller programs.
//
// Outline of the process:
// - inputs to the generation are the program and options
// - options control multiple aspects of the resulting C program,
//   like if we want a multi-threaded program or a single-threaded,
//   what type of sandbox we want to use, if we want to setup net devices or not, etc
// - we use actual executor sources as the base
// - gen.go takes all executor/common*.h headers and bundles them into generated.go
// - during generation we tear executor headers apart and take only the bits
//   we need for the current program/options, this is done by running C preprocessor
//   with particular set of defines so that the preprocessor removes unneeded
//   #ifdef SYZ_FOO sections
// - then we generate actual syscall calls with the given arguments
//   based on the binary "encodingexec" representation of the program
//   (the same representation executor uses for interpretation)
// - then we glue it all together
// - as the last step we run some text post-processing on the resulting source code:
//   remove debug calls, replace exitf/fail with exit, hoist/sort/dedup includes,
//   remove duplicate empty lines, etc
package csource

import (
	"bytes"
	"fmt"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/sys/targets"
)

// Write generates C source for program p based on the provided options opt.
func Write(p *prog.Prog, opts Options) ([]byte, error) {
	if err := opts.Check(p.Target.OS); err != nil {
		return nil, fmt.Errorf("csource: invalid opts: %v", err)
	}
	ctx := &context{
		p:         p,
		opts:      opts,
		target:    p.Target,
		sysTarget: targets.Get(p.Target.OS, p.Target.Arch),
		calls:     make(map[string]uint64),
	}
	return ctx.generateSource()
}

type context struct {
	p         *prog.Prog
	opts      Options
	target    *prog.Target
	sysTarget *targets.Target
	calls     map[string]uint64 // CallName -> NR
}

func (ctx *context) generateSource() ([]byte, error) {
	ctx.filterCalls()
	calls, vars, err := ctx.generateProgCalls(ctx.p, ctx.opts.Trace)
	if err != nil {
		return nil, err
	}

	mmapProg := ctx.p.Target.DataMmapProg()
	mmapCalls, _, err := ctx.generateProgCalls(mmapProg, false)
	if err != nil {
		return nil, err
	}

	for _, c := range append(mmapProg.Calls, ctx.p.Calls...) {
		ctx.calls[c.Meta.CallName] = c.Meta.NR
	}

	varsBuf := new(bytes.Buffer)
	if len(vars) != 0 {
		fmt.Fprintf(varsBuf, "uint64 r[%v] = {", len(vars))
		for i, v := range vars {
			if i != 0 {
				fmt.Fprintf(varsBuf, ", ")
			}
			fmt.Fprintf(varsBuf, "0x%x", v)
		}
		fmt.Fprintf(varsBuf, "};\n")
	}

	sandboxFunc := "loop();"
	if ctx.opts.Sandbox != "" {
		sandboxFunc = "do_sandbox_" + ctx.opts.Sandbox + "();"
	}
	replacements := map[string]string{
		"PROCS":           fmt.Sprint(ctx.opts.Procs),
		"REPEAT_TIMES":    fmt.Sprint(ctx.opts.RepeatTimes),
		"NUM_CALLS":       fmt.Sprint(len(ctx.p.Calls)),
		"MMAP_DATA":       strings.Join(mmapCalls, ""),
		"SYSCALL_DEFINES": ctx.generateSyscallDefines(),
		"SANDBOX_FUNC":    sandboxFunc,
		"RESULTS":         varsBuf.String(),
		"SYSCALLS":        ctx.generateSyscalls(calls, len(vars) != 0),
	}
	if !ctx.opts.Threaded && !ctx.opts.Repeat && ctx.opts.Sandbox == "" {
		// This inlines syscalls right into main for the simplest case.
		replacements["SANDBOX_FUNC"] = replacements["SYSCALLS"]
		replacements["SYSCALLS"] = "unused"
	}
	timeouts := ctx.sysTarget.Timeouts(ctx.opts.Slowdown)
	replacements["PROGRAM_TIMEOUT_MS"] = fmt.Sprint(int(timeouts.Program / time.Millisecond))
	timeoutExpr := fmt.Sprint(int(timeouts.Syscall / time.Millisecond))
	for i, call := range ctx.p.Calls {
		if timeout := call.Meta.Attrs.Timeout; timeout != 0 {
			timeoutExpr += fmt.Sprintf(" + (call == %v ? %v : 0)", i, timeout*uint64(timeouts.Scale))
		}
	}
	replacements["CALL_TIMEOUT_MS"] = timeoutExpr
	result, err := createCommonHeader(ctx.p, mmapProg, replacements, ctx.opts)
	if err != nil {
		return nil, err
	}
	const header = "// autogenerated by syzkaller (https://github.com/google/syzkaller)\n\n"
	result = append([]byte(header), result...)
	result = ctx.postProcess(result)
	return result, nil
}

// This is a kludge, but we keep it here until a better approach is implemented.
// TODO: untie syz_emit_ethernet/syz_extract_tcp_res and NetInjection. And also
// untie VhciInjection and syz_emit_vhci. Then we could remove this method.
func (ctx *context) filterCalls() {
	p := ctx.p
	for i := 0; i < len(p.Calls); {
		call := p.Calls[i]
		callName := call.Meta.CallName
		emitCall := (ctx.opts.NetInjection ||
			callName != "syz_emit_ethernet" &&
				callName != "syz_extract_tcp_res") &&
			(ctx.opts.VhciInjection || callName != "syz_emit_vhci")
		if emitCall {
			i++
			continue
		}
		// Remove the call.
		if ctx.p == p {
			// We lazily clone the program to avoid unnecessary copying.
			p = ctx.p.Clone()
		}
		p.RemoveCall(i)
	}
	ctx.p = p
}

func (ctx *context) generateSyscalls(calls []string, hasVars bool) string {
{'fraction_non_alphanumeric': 0.13081060315021129, 'fraction_numerical': 0.00384172109104879, 'mean_word_length': 3.4806368330464714, 'pattern_counts': {'":': 16, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1840196', 'n_tokens_mistral': 3770, 'n_tokens_neox': 3551, 'n_words': 1049}