<commit_before>package pointer_test

// This test uses 'expectation' comments embedded within testdata/*.go
// files to specify the expected pointer analysis behaviour.
// See below for grammar.

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"strings"
	"testing"

	"code.google.com/p/go.tools/go/types"
	"code.google.com/p/go.tools/go/types/typemap"
	"code.google.com/p/go.tools/importer"
	"code.google.com/p/go.tools/pointer"
	"code.google.com/p/go.tools/ssa"
)

var inputs = []string{
	// Currently debugging:
	// "testdata/tmp.go",

	// Working:
	"testdata/another.go",
	"testdata/arrays.go",
	"testdata/channels.go",
	"testdata/context.go",
	"testdata/conv.go",
	"testdata/flow.go",
	"testdata/fmtexcerpt.go",
	"testdata/func.go",
	"testdata/hello.go",
	"testdata/interfaces.go",
	"testdata/maps.go",
	"testdata/panic.go",
	"testdata/recur.go",
	"testdata/structs.go",
	"testdata/a_test.go",

	// TODO(adonovan): get these tests (of reflection) passing.
	// (The tests are mostly sound since they were used for a
	// previous implementation.)
	// "testdata/funcreflect.go",
	// "testdata/arrayreflect.go",
	// "testdata/chanreflect.go",
	// "testdata/finalizer.go",
	// "testdata/reflect.go",
	// "testdata/mapreflect.go",
	// "testdata/structreflect.go",
}

// Expectation grammar:
//
// @calls f -> g
//
//   A 'calls' expectation asserts that edge (f, g) appears in the
//   callgraph.  f and g are notated as per Function.String(), which
//   may contain spaces (e.g. promoted method in anon struct).
//
// @pointsto a | b | c
//
//   A 'pointsto' expectation asserts that the points-to set of its
//   operand contains exactly the set of labels {a,b,c} notated as per
//   labelString.
//
//   A 'pointsto' expectation must appear on the same line as a
//   print(x) statement; the expectation's operand is x.
//
//   If one of the strings is "...", the expectation asserts that the
//   points-to set at least the other labels.
//
//   We use '|' because label names may contain spaces, e.g.  methods
//   of anonymous structs.
//
//   From a theoretical perspective, concrete types in interfaces are
//   labels too, but they are represented differently and so have a
//   different expectation, @concrete, below.
//
// @concrete t | u | v
//
//   A 'concrete' expectation asserts that the set of possible dynamic
//   types of its interface operand is exactly {t,u,v}, notated per
//   go/types.Type.String(). In other words, it asserts that the type
//   component of the interface may point to that set of concrete type
//   literals.
//
//   A 'concrete' expectation must appear on the same line as a
//   print(x) statement; the expectation's operand is x.
//
//   If one of the strings is "...", the expectation asserts that the
//   interface's type may point to at least the  other concrete types.
//
//   We use '|' because type names may contain spaces.
//
// @warning "regexp"
//
//   A 'warning' expectation asserts that the analysis issues a
//   warning that matches the regular expression within the string
//   literal.
//
// @line id
//
//   A line directive associates the name "id" with the current
//   file:line.  The string form of labels will use this id instead of
//   a file:line, making @pointsto expectations more robust against
//   perturbations in the source file.
//   (NB, anon functions still include line numbers.)
//
type expectation struct {
	kind     string // "pointsto" | "concrete" | "calls" | "warning"
	filename string
	linenum  int // source line number, 1-based
	args     []string
	types    []types.Type // for concrete
}

func (e *expectation) String() string {
	return fmt.Sprintf("@%s[%s]", e.kind, strings.Join(e.args, " | "))
}

func (e *expectation) errorf(format string, args ...interface{}) {
	fmt.Printf("%s:%d: ", e.filename, e.linenum)
	fmt.Printf(format, args...)
	fmt.Println()
}

func (e *expectation) needsProbe() bool {
	return e.kind == "pointsto" || e.kind == "concrete"
}

// A record of a call to the built-in print() function.  Used for testing.
type probe struct {
	instr *ssa.CallCommon
	arg0  pointer.Pointer // first argument to print
}

// Find probe (call to print(x)) of same source
// file/line as expectation.
func findProbe(prog *ssa.Program, probes []probe, e *expectation) *probe {
	for _, p := range probes {
		pos := prog.Fset.Position(p.instr.Pos())
		if pos.Line == e.linenum && pos.Filename == e.filename {
			// TODO(adonovan): send this to test log (display only on failure).
			// fmt.Printf("%s:%d: info: found probe for %s: %s\n",
			// 	e.filename, e.linenum, e, p.arg0) // debugging
			return &p
		}
	}
	return nil // e.g. analysis didn't reach this call
}

func doOneInput(input, filename string) bool {
	impctx := &importer.Config{Loader: importer.MakeGoBuildLoader(nil)}
	imp := importer.New(impctx)

	// Parsing.
	f, err := parser.ParseFile(imp.Fset, filename, input, parser.DeclarationErrors)
	if err != nil {
		// TODO(adonovan): err is a scanner error list;
		// display all errors not just first?
		fmt.Println(err.Error())
		return false
	}

	// Type checking.
	info := imp.CreateSourcePackage("main", []*ast.File{f})
	if info.Err != nil {
		fmt.Println(info.Err.Error())
		return false
	}

	// SSA creation + building.
	prog := ssa.NewProgram(imp.Fset, ssa.SanityCheckFunctions)
	for _, info := range imp.Packages {
		prog.CreatePackage(info)
	}
	prog.BuildAll()

	mainpkg := prog.Package(info.Pkg)
	ptrmain := mainpkg // main package for the pointer analysis
	if mainpkg.Func("main") == nil {
		// No main function; assume it's a test.
		ptrmain = prog.CreateTestMainPackage([]*ssa.Package{mainpkg})
		fmt.Printf("%s: synthesized testmain package for test.\n", imp.Fset.Position(f.Package))
	}

	ok := true

	lineMapping := make(map[string]string) // maps "file:line" to @line tag

	// Parse expectations in this input.
	var exps []*expectation
	re := regexp.MustCompile("// *@([a-z]*) *(.*)$")
	lines := strings.Split(input, "\n")
	for linenum, line := range lines {
		linenum++ // make it 1-based
		if matches := re.FindAllStringSubmatch(line, -1); matches != nil {
			match := matches[0]
			kind, rest := match[1], match[2]
			e := &expectation{kind: kind, filename: filename, linenum: linenum}

			if kind == "line" {
				if rest == "" {
					ok = false
					e.errorf("@%s expectation requires identifier", kind)
				} else {
					lineMapping[fmt.Sprintf("%s:%d", filename, linenum)] = rest
				}
				continue
			}

			if e.needsProbe() && !strings.Contains(line, "print(") {
				ok = false
				e.errorf("@%s expectation must follow call to print(x)", kind)<commit_msg>go.tools/pointer: fix build breakage.<commit_after>package pointer_test

// This test uses 'expectation' comments embedded within testdata/*.go
// files to specify the expected pointer analysis behaviour.
// See below for grammar.

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"strings"
	"testing"

	"code.google.com/p/go.tools/go/types"
	"code.google.com/p/go.tools/go/types/typemap"
	"code.google.com/p/go.tools/importer"
	"code.google.com/p/go.tools/pointer"
	"code.google.com/p/go.tools/ssa"
)

var inputs = []string{
	// Currently debugging:
	// "testdata/tmp.go",

	// Working:
	"testdata/another.go",
	"testdata/arrays.go",
	"testdata/channels.go",
	"testdata/context.go",
	"testdata/conv.go",
	"testdata/flow.go",
	"testdata/fmtexcerpt.go",
	"testdata/func.go",
	"testdata/hello.go",
	"testdata/interfaces.go",
	"testdata/maps.go",
	"testdata/panic.go",
	"testdata/recur.go",
	"testdata/structs.go",
	"testdata/a_test.go",

	// TODO(adonovan): get these tests (of reflection) passing.
	// (The tests are mostly sound since they were used for a
	// previous implementation.)
	// "testdata/funcreflect.go",
	// "testdata/arrayreflect.go",
	// "testdata/chanreflect.go",
	// "testdata/finalizer.go",
	// "testdata/reflect.go",
	// "testdata/mapreflect.go",
	// "testdata/structreflect.go",
}

// Expectation grammar:
//
// @calls f -> g
//
//   A 'calls' expectation asserts that edge (f, g) appears in the
//   callgraph.  f and g are notated as per Function.String(), which
//   may contain spaces (e.g. promoted method in anon struct).
//
// @pointsto a | b | c
//
//   A 'pointsto' expectation asserts that the points-to set of its
//   operand contains exactly the set of labels {a,b,c} notated as per
//   labelString.
//
//   A 'pointsto' expectation must appear on the same line as a
//   print(x) statement; the expectation's operand is x.
//
//   If one of the strings is "...", the expectation asserts that the
//   points-to set at least the other labels.
//
//   We use '|' because label names may contain spaces, e.g.  methods
//   of anonymous structs.
//
//   From a theoretical perspective, concrete types in interfaces are
//   labels too, but they are represented differently and so have a
//   different expectation, @concrete, below.
//
// @concrete t | u | v
//
//   A 'concrete' expectation asserts that the set of possible dynamic
//   types of its interface operand is exactly {t,u,v}, notated per
//   go/types.Type.String(). In other words, it asserts that the type
//   component of the interface may point to that set of concrete type
//   literals.
//
//   A 'concrete' expectation must appear on the same line as a
//   print(x) statement; the expectation's operand is x.
//
//   If one of the strings is "...", the expectation asserts that the
//   interface's type may point to at least the  other concrete types.
//
//   We use '|' because type names may contain spaces.
//
// @warning "regexp"
//
//   A 'warning' expectation asserts that the analysis issues a
//   warning that matches the regular expression within the string
//   literal.
//
// @line id
//
//   A line directive associates the name "id" with the current
//   file:line.  The string form of labels will use this id instead of
//   a file:line, making @pointsto expectations more robust against
//   perturbations in the source file.
//   (NB, anon functions still include line numbers.)
//
type expectation struct {
	kind     string // "pointsto" | "concrete" | "calls" | "warning"
	filename string
	linenum  int // source line number, 1-based
	args     []string
	types    []types.Type // for concrete
}

func (e *expectation) String() string {
	return fmt.Sprintf("@%s[%s]", e.kind, strings.Join(e.args, " | "))
}

func (e *expectation) errorf(format string, args ...interface{}) {
	fmt.Printf("%s:%d: ", e.filename, e.linenum)
	fmt.Printf(format, args...)
	fmt.Println()
}

func (e *expectation) needsProbe() bool {
	return e.kind == "pointsto" || e.kind == "concrete"
}

// A record of a call to the built-in print() function.  Used for testing.
type probe struct {
	instr *ssa.CallCommon
	arg0  pointer.Pointer // first argument to print
}

// Find probe (call to print(x)) of same source
// file/line as expectation.
func findProbe(prog *ssa.Program, probes []probe, e *expectation) *probe {
	for _, p := range probes {
		pos := prog.Fset.Position(p.instr.Pos())
		if pos.Line == e.linenum && pos.Filename == e.filename {
			// TODO(adonovan): send this to test log (display only on failure).
			// fmt.Printf("%s:%d: info: found probe for %s: %s\n",
			// 	e.filename, e.linenum, e, p.arg0) // debugging
			return &p
		}
	}
	return nil // e.g. analysis didn't reach this call
}

func doOneInput(input, filename string) bool {
	impctx := &importer.Config{Loader: importer.MakeGoBuildLoader(nil)}
	imp := importer.New(impctx)

	// Parsing.
	f, err := parser.ParseFile(imp.Fset, filename, input, parser.DeclarationErrors)
	if err != nil {
		// TODO(adonovan): err is a scanner error list;
		// display all errors not just first?
		fmt.Println(err.Error())
		return false
	}

	// Type checking.
	info := imp.CreateSourcePackage("main", []*ast.File{f})
	if info.Err != nil {
		fmt.Println(info.Err.Error())
		return false
	}

	// SSA creation + building.
	prog := ssa.NewProgram(imp.Fset, ssa.SanityCheckFunctions)
	for _, info := range imp.Packages {
		prog.CreatePackage(info)
	}
	prog.BuildAll()

	mainpkg := prog.Package(info.Pkg)
	ptrmain := mainpkg // main package for the pointer analysis
	if mainpkg.Func("main") == nil {
		// No main function; assume it's a test.
		mainpkg.CreateTestMainFunction()
		fmt.Printf("%s: synthesized testmain package for test.\n", imp.Fset.Position(f.Package))
	}

	ok := true

	lineMapping := make(map[string]string) // maps "file:line" to @line tag

	// Parse expectations in this input.
	var exps []*expectation
	re := regexp.MustCompile("// *@([a-z]*) *(.*)$")
	lines := strings.Split(input, "\n")
	for linenum, line := range lines {
		linenum++ // make it 1-based
		if matches := re.FindAllStringSubmatch(line, -1); matches != nil {
			match := matches[0]
			kind, rest := match[1], match[2]
			e := &expectation{kind: kind, filename: filename, linenum: linenum}

			if kind == "line" {
				if rest == "" {
					ok = false
					e.errorf("@%s expectation requires identifier", kind)
				} else {
					lineMapping[fmt.Sprintf("%s:%d", filename, linenum)] = rest
				}
				continue
			}

			if e.needsProbe() && !strings.Contains(line, "print(") {
				ok = false
				e.errorf("@%s expectation must follow call to print(x)", kind)
{'fraction_non_alphanumeric': 0.1495291902071563, 'fraction_numerical': 0.0012052730696798493, 'mean_word_length': 3.198608475648324, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2847702', 'n_tokens_mistral': 4638, 'n_tokens_neox': 4420, 'n_words': 1387}