go/packages/packagestest: add marker support

This adds a library that allows markers and actions inside comments in go source
files.
It then also adds an easy way to use that library for tests using packagestest.Expect
This is used to easily write code inspection and manipulation tests in a
language that is common to all tests.

Change-Id: I755caaad1557c9b8779ad9ecda2b3309550d6976
Reviewed-on: https://go-review.googlesource.com/c/142998
Run-TryBot: Ian Cottrell <iancottrell@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Michael Matloob <matloob@golang.org>

go/expect/expect.go

@@ -0,0 +1,149 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+Package expect provides support for interpreting structured comments in Go
+source code as test expectations.
+
+This is primarily intended for writing tests of things that process Go source
+files, although it does not directly depend on the testing package.
+
+Collect notes with the Extract or Parse functions, and use the
+MatchBefore function to find matches within the lines the comments were on.
+
+The interpretation of the notes depends on the application.
+For example, the test suite for a static checking tool might
+use a @diag note to indicate an expected diagnostic:
+
+   fmt.Printf("%s", 1) //@ diag("%s wants a string, got int")
+
+By contrast, the test suite for a source code navigation tool
+might use notes to indicate the positions of features of
+interest, the actions to be performed by the test,
+and their expected outcomes:
+
+   var x = 1 //@ x_decl
+   ...
+   print(x) //@ definition("x", x_decl)
+   print(x) //@ typeof("x", "int")
+
+
+Note comment syntax
+
+Note comments always start with the special marker @, which must be the
+very first character after the comment opening pair, so //@ or /*@ with no
+spaces.
+
+This is followed by a comma separated list of notes.
+
+A note always starts with an identifier, which is optionally followed by an
+argument list. The argument list is surrounded with parentheses and contains a
+comma-separated list of arguments.
+The empty parameter list and the missing parameter list are distinguishable if
+needed; they result in a nil or an empty list in the Args parameter respectively.
+
+Arguments are either identifiers or literals.
+The literals supported are the basic value literals, of string, float, integer
+true, false or nil. All the literals match the standard go conventions, with
+all bases of integers, and both quote and backtick strings.
+There is one extra literal type, which is a string literal preceded by the
+identifier "re" which is compiled to a regular expression.
+*/
+package expect
+
+import (
+	"bytes"
+	"fmt"
+	"go/token"
+	"regexp"
+)
+
+// Note is a parsed note from an expect comment.
+// It knows the position of the start of the comment, and the name and
+// arguments that make up the note.
+type Note struct {
+	Pos  token.Pos     // The position at which the note identifier appears
+	Name string        // the name associated with the note
+	Args []interface{} // the arguments for the note
+}
+
+// ReadFile  is the type of a function that can provide file contents for a
+// given filename.
+// This is used in MatchBefore to look up the content of the file in order to
+// find the line to match the pattern against.
+type ReadFile func(filename string) ([]byte, error)
+
+// MatchBefore attempts to match a pattern in the line before the supplied pos.
+// It uses the FileSet and the ReadFile to work out the contents of the line
+// that end is part of, and then matches the pattern against the content of the
+// start of that line up to the supplied position.
+// The pattern may be either a simple string, []byte or a *regexp.Regexp.
+// MatchBefore returns the range of the line that matched the pattern, and
+// invalid positions if there was no match, or an error if the line could not be
+// found.
+func MatchBefore(fset *token.FileSet, readFile ReadFile, end token.Pos, pattern interface{}) (token.Pos, token.Pos, error) {
+	f := fset.File(end)
+	content, err := readFile(f.Name())
+	if err != nil {
+		return token.NoPos, token.NoPos, fmt.Errorf("invalid file: %v", err)
+	}
+	position := f.Position(end)
+	startOffset := f.Offset(lineStart(f, position.Line))
+	endOffset := f.Offset(end)
+	line := content[startOffset:endOffset]
+	matchStart, matchEnd := -1, -1
+	switch pattern := pattern.(type) {
+	case string:
+		bytePattern := []byte(pattern)
+		matchStart = bytes.Index(line, bytePattern)
+		if matchStart >= 0 {
+			matchEnd = matchStart + len(bytePattern)
+		}
+	case []byte:
+		matchStart = bytes.Index(line, pattern)
+		if matchStart >= 0 {
+			matchEnd = matchStart + len(pattern)
+		}
+	case *regexp.Regexp:
+		match := pattern.FindIndex(line)
+		if len(match) > 0 {
+			matchStart = match[0]
+			matchEnd = match[1]
+		}
+	}
+	if matchStart < 0 {
+		return token.NoPos, token.NoPos, nil
+	}
+	return f.Pos(startOffset + matchStart), f.Pos(startOffset + matchEnd), nil
+}
+
+// this functionality was borrowed from the analysisutil package
+func lineStart(f *token.File, line int) token.Pos {
+	// Use binary search to find the start offset of this line.
+	//
+	// TODO(adonovan): eventually replace this function with the
+	// simpler and more efficient (*go/token.File).LineStart, added
+	// in go1.12.
+
+	min := 0        // inclusive
+	max := f.Size() // exclusive
+	for {
+		offset := (min + max) / 2
+		pos := f.Pos(offset)
+		posn := f.Position(pos)
+		if posn.Line == line {
+			return pos - (token.Pos(posn.Column) - 1)
+		}
+
+		if min+1 >= max {
+			return token.NoPos
+		}
+
+		if posn.Line < line {
+			min = offset
+		} else {
+			max = offset
+		}
+	}
+}

go/expect/expect_test.go

@@ -0,0 +1,135 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package expect_test
+
+import (
+	"bytes"
+	"go/token"
+	"io/ioutil"
+	"testing"
+
+	"golang.org/x/tools/go/expect"
+)
+
+func TestMarker(t *testing.T) {
+	const filename = "testdata/test.go"
+	content, err := ioutil.ReadFile(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	const expectNotes = 11
+	expectMarkers := map[string]string{
+		"αSimpleMarker": "α",
+		"OffsetMarker":  "β",
+		"RegexMarker":   "γ",
+		"εMultiple":     "ε",
+		"ζMarkers":      "ζ",
+		"ηBlockMarker":  "η",
+		"Declared":      "η",
+		"Comment":       "ι",
+		"NonIdentifier": "+",
+	}
+	expectChecks := map[string][]interface{}{
+		"αSimpleMarker": nil,
+		"StringAndInt":  []interface{}{"Number %d", int64(12)},
+		"Bool":          []interface{}{true},
+	}
+
+	readFile := func(string) ([]byte, error) { return content, nil }
+	markers := make(map[string]token.Pos)
+	for name, tok := range expectMarkers {
+		offset := bytes.Index(content, []byte(tok))
+		markers[name] = token.Pos(offset + 1)
+		end := bytes.Index(content[offset+1:], []byte(tok))
+		if end > 0 {
+			markers[name+"@"] = token.Pos(offset + end + 2)
+		}
+	}
+
+	fset := token.NewFileSet()
+	notes, err := expect.Parse(fset, filename, nil)
+	if err != nil {
+		t.Fatalf("Failed to extract notes: %v", err)
+	}
+	if len(notes) != expectNotes {
+		t.Errorf("Expected %v notes, got %v", expectNotes, len(notes))
+	}
+	for _, n := range notes {
+		switch {
+		case n.Args == nil:
+			// A //@foo note associates the name foo with the position of the
+			// first match of "foo" on the current line.
+			checkMarker(t, fset, readFile, markers, n.Pos, n.Name, n.Name)
+		case n.Name == "mark":
+			// A //@mark(name, "pattern") note associates the specified name
+			// with the position on the first match of pattern on the current line.
+			if len(n.Args) != 2 {
+				t.Errorf("%v: expected 2 args to mark, got %v", fset.Position(n.Pos), len(n.Args))
+				continue
+			}
+			ident, ok := n.Args[0].(expect.Identifier)
+			if !ok {
+				t.Errorf("%v: identifier, got %T", fset.Position(n.Pos), n.Args[0])
+				continue
+			}
+			checkMarker(t, fset, readFile, markers, n.Pos, string(ident), n.Args[1])
+
+		case n.Name == "check":
+			// A //@check(args, ...) note specifies some hypothetical action to
+			// be taken by the test driver and its expected outcome.
+			// In this test, the action is to compare the arguments
+			// against expectChecks.
+			if len(n.Args) < 1 {
+				t.Errorf("%v: expected 1 args to check, got %v", fset.Position(n.Pos), len(n.Args))
+				continue
+			}
+			ident, ok := n.Args[0].(expect.Identifier)
+			if !ok {
+				t.Errorf("%v: identifier, got %T", fset.Position(n.Pos), n.Args[0])
+				continue
+			}
+			args, ok := expectChecks[string(ident)]
+			if !ok {
+				t.Errorf("%v: unexpected check %v", fset.Position(n.Pos), ident)
+				continue
+			}
+			if len(n.Args) != len(args)+1 {
+				t.Errorf("%v: expected %v args to check, got %v", fset.Position(n.Pos), len(args)+1, len(n.Args))
+				continue
+			}
+			for i, got := range n.Args[1:] {
+				if args[i] != got {
+					t.Errorf("%v: arg %d expected %v, got %v", fset.Position(n.Pos), i, args[i], got)
+				}
+			}
+		default:
+			t.Errorf("Unexpected note %v at %v", n.Name, fset.Position(n.Pos))
+		}
+	}
+}
+
+func checkMarker(t *testing.T, fset *token.FileSet, readFile expect.ReadFile, markers map[string]token.Pos, pos token.Pos, name string, pattern interface{}) {
+	start, end, err := expect.MatchBefore(fset, readFile, pos, pattern)
+	if err != nil {
+		t.Errorf("%v: MatchBefore failed: %v", fset.Position(pos), err)
+		return
+	}
+	if start == token.NoPos {
+		t.Errorf("%v: Pattern %v did not match", fset.Position(pos), pattern)
+		return
+	}
+	expectStart, ok := markers[name]
+	if !ok {
+		t.Errorf("%v: unexpected marker %v", fset.Position(pos), name)
+		return
+	}
+	if start != expectStart {
+		t.Errorf("%v: Expected %v got %v", fset.Position(pos), fset.Position(expectStart), fset.Position(start))
+	}
+	if expectEnd, ok := markers[name+"@"]; ok && end != expectEnd {
+		t.Errorf("%v: Expected end %v got %v", fset.Position(pos), fset.Position(expectEnd), fset.Position(end))
+	}
+}

go/expect/extract.go

@@ -0,0 +1,221 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package expect
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"regexp"
+	"strconv"
+	"strings"
+	"text/scanner"
+)
+
+const (
+	commentStart = "@"
+)
+
+// Identifier is the type for an identifier in an Note argument list.
+type Identifier string
+
+// Parse collects all the notes present in a file.
+// If content is nil, the filename specified is read and parsed, otherwise the
+// content is used and the filename is used for positions and error messages.
+// Each comment whose text starts with @ is parsed as a comma-separated
+// sequence of notes.
+// See the package documentation for details about the syntax of those
+// notes.
+func Parse(fset *token.FileSet, filename string, content []byte) ([]*Note, error) {
+	var src interface{}
+	if content != nil {
+		src = content
+	}
+	// TODO: We should write this in terms of the scanner.
+	// there are ways you can break the parser such that it will not add all the
+	// comments to the ast, which may result in files where the tests are silently
+	// not run.
+	file, err := parser.ParseFile(fset, filename, src, parser.ParseComments)
+	if file == nil {
+		return nil, err
+	}
+	return Extract(fset, file)
+}
+
+// Extract collects all the notes present in an AST.
+// Each comment whose text starts with @ is parsed as a comma-separated
+// sequence of notes.
+// See the package documentation for details about the syntax of those
+// notes.
+func Extract(fset *token.FileSet, file *ast.File) ([]*Note, error) {
+	var notes []*Note
+	for _, g := range file.Comments {
+		for _, c := range g.List {
+			text := c.Text
+			if strings.HasPrefix(text, "/*") {
+				text = strings.TrimSuffix(text, "*/")
+			}
+			text = text[2:] // remove "//" or "/*" prefix
+			if !strings.HasPrefix(text, commentStart) {
+				continue
+			}
+			text = text[len(commentStart):]
+			parsed, err := parse(fset, c.Pos()+4, text)
+			if err != nil {
+				return nil, err
+			}
+			notes = append(notes, parsed...)
+		}
+	}
+	return notes, nil
+}
+
+func parse(fset *token.FileSet, base token.Pos, text string) ([]*Note, error) {
+	var scanErr error
+	s := new(scanner.Scanner).Init(strings.NewReader(text))
+	s.Mode = scanner.GoTokens
+	s.Error = func(s *scanner.Scanner, msg string) {
+		scanErr = fmt.Errorf("%v:%s", fset.Position(base+token.Pos(s.Position.Offset)), msg)
+	}
+	notes, err := parseComment(s)
+	if err != nil {
+		return nil, fmt.Errorf("%v:%s", fset.Position(base+token.Pos(s.Position.Offset)), err)
+	}
+	if scanErr != nil {
+		return nil, scanErr
+	}
+	for _, n := range notes {
+		n.Pos += base
+	}
+	return notes, nil
+}
+
+func parseComment(s *scanner.Scanner) ([]*Note, error) {
+	var notes []*Note
+	for {
+		n, err := parseNote(s)
+		if err != nil {
+			return nil, err
+		}
+		notes = append(notes, n)
+		tok := s.Scan()
+		switch tok {
+		case ',':
+			// continue
+		case scanner.EOF:
+			return notes, nil
+		default:
+			return nil, fmt.Errorf("unexpected %s parsing comment", scanner.TokenString(tok))
+		}
+	}
+}
+
+func parseNote(s *scanner.Scanner) (*Note, error) {
+	if tok := s.Scan(); tok != scanner.Ident {
+		return nil, fmt.Errorf("expected identifier, got %s", scanner.TokenString(tok))
+	}
+	n := &Note{
+		Pos:  token.Pos(s.Position.Offset),
+		Name: s.TokenText(),
+	}
+	switch s.Peek() {
+	case ',', scanner.EOF:
+		// no argument list present
+		return n, nil
+	case '(':
+		// parse the argument list
+		if tok := s.Scan(); tok != '(' {
+			return nil, fmt.Errorf("expected ( got %s", scanner.TokenString(tok))
+		}
+		// special case the empty argument list
+		if s.Peek() == ')' {
+			if tok := s.Scan(); tok != ')' {
+				return nil, fmt.Errorf("expected ) got %s", scanner.TokenString(tok))
+			}
+			n.Args = []interface{}{} // @name() is represented by a non-nil empty slice.
+			return n, nil
+		}
+		// handle a normal argument list
+		for {
+			arg, err := parseArgument(s)
+			if err != nil {
+				return nil, err
+			}
+			n.Args = append(n.Args, arg)
+			switch s.Peek() {
+			case ')':
+				if tok := s.Scan(); tok != ')' {
+					return nil, fmt.Errorf("expected ) got %s", scanner.TokenString(tok))
+				}
+				return n, nil
+			case ',':
+				if tok := s.Scan(); tok != ',' {
+					return nil, fmt.Errorf("expected , got %s", scanner.TokenString(tok))
+				}
+				// continue
+			default:
+				return nil, fmt.Errorf("unexpected %s parsing argument list", scanner.TokenString(s.Scan()))
+			}
+		}
+	default:
+		return nil, fmt.Errorf("unexpected %s parsing note", scanner.TokenString(s.Scan()))
+	}
+}
+
+func parseArgument(s *scanner.Scanner) (interface{}, error) {
+	tok := s.Scan()
+	switch tok {
+	case scanner.Ident:
+		v := s.TokenText()
+		switch v {
+		case "true":
+			return true, nil
+		case "false":
+			return false, nil
+		case "nil":
+			return nil, nil
+		case "re":
+			tok := s.Scan()
+			switch tok {
+			case scanner.String, scanner.RawString:
+				pattern, _ := strconv.Unquote(s.TokenText()) // can't fail
+				re, err := regexp.Compile(pattern)
+				if err != nil {
+					return nil, fmt.Errorf("invalid regular expression %s: %v", pattern, err)
+				}
+				return re, nil
+			default:
+				return nil, fmt.Errorf("re must be followed by string, got %s", scanner.TokenString(tok))
+			}
+		default:
+			return Identifier(v), nil
+		}
+
+	case scanner.String, scanner.RawString:
+		v, _ := strconv.Unquote(s.TokenText()) // can't fail
+		return v, nil
+
+	case scanner.Int:
+		v, err := strconv.ParseInt(s.TokenText(), 0, 0)
+		if err != nil {
+			return nil, fmt.Errorf("cannot convert %v to int: %v", s.TokenText(), err)
+		}
+		return v, nil
+
+	case scanner.Float:
+		v, err := strconv.ParseFloat(s.TokenText(), 64)
+		if err != nil {
+			return nil, fmt.Errorf("cannot convert %v to float: %v", s.TokenText(), err)
+		}
+		return v, nil
+
+	case scanner.Char:
+		return nil, fmt.Errorf("unexpected char literal %s", s.TokenText())
+
+	default:
+		return nil, fmt.Errorf("unexpected %s parsing argument", scanner.TokenString(tok))
+	}
+}

go/expect/testdata/test.go

@@ -0,0 +1,30 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fake1
+
+// The greek letters in this file mark points we use for marker tests.
+// We use unique markers so we can make the tests stable against changes to
+// this file.
+
+const (
+	_                   int = iota
+	αSimpleMarkerα          //@αSimpleMarker
+	offsetββMarker          //@mark(OffsetMarker, "β")
+	regexγMaγrker           //@mark(RegexMarker, re`\p{Greek}Ma`)
+	εMultipleεζMarkersζ     //@εMultiple,ζMarkers
+	ηBlockMarkerη           /*@ηBlockMarker*/
+)
+
+/*Marker ι inside ι a comment*/ //@mark(Comment,"ι inside ")
+
+func someFunc(a, b int) int {
+	// The line below must be the first occurrence of the plus operator
+	return a + b + 1 //@mark(NonIdentifier, re`\+[^\+]*`)
+}
+
+// And some extra checks for interesting action parameters
+//@check(αSimpleMarker)
+//@check(StringAndInt, "Number %d", 12)
+//@check(Bool, true)

go/packages/packagestest/expect.go

@@ -0,0 +1,297 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packagestest
+
+import (
+	"fmt"
+	"go/token"
+	"reflect"
+	"regexp"
+	"strings"
+
+	"golang.org/x/tools/go/expect"
+)
+
+// Expect invokes the supplied methods for all expectation comments found in
+// the exported source files.
+//
+// All exported go source files are parsed to collect the expectation
+// expressions.
+// See the documentation for expect.Parse for how the expectations are collected
+// and parsed.
+//
+// The methods are supplied as a map of name to function, and those functions
+// will be matched against the expectations by name.
+// Markers with no matching function will be skipped, and functions with no
+// matching markers will not be invoked.
+// As a special case expectations for the mark function will be processed and
+// the names can then be used to identify positions in files for all other
+// methods invoked.
+//
+// Method invocation
+//
+// When invoking a method the expressions in the parameter list need to be
+// converted to values to be passed to the method.
+// There are a very limited set of types the arguments are allowed to be.
+//   expect.Comment : passed the Comment instance being evaluated.
+//   string : can be supplied either a string literal or an identifier.
+//   int : can only be supplied an integer literal.
+//   token.Pos : has a file position calculated as described below.
+//   token.Position : has a file position calculated as described below.
+//
+// Position calculation
+//
+// There is some extra handling when a parameter is being coerced into a
+// token.Pos or token.Position type argument.
+//
+// If the parameter is an identifier, it will be treated as the name of an
+// marker to look up (as if markers were global variables). These markers
+// are the results of all "mark" expectations, where the first parameter is
+// the name of the marker and the second is the position of the marker.
+//
+// If it is a string or regular expression, then it will be passed to
+// expect.MatchBefore to look up a match in the line at which it was declared.
+//
+// It is safe to call this repeatedly with different method sets, but it is
+// not safe to call it concurrently.
+func (e *Exported) Expect(methods map[string]interface{}) error {
+	if e.notes == nil {
+		notes := []*expect.Note{}
+		for _, module := range e.written {
+			for _, filename := range module {
+				if !strings.HasSuffix(filename, ".go") {
+					continue
+				}
+				l, err := expect.Parse(e.fset, filename, nil)
+				if err != nil {
+					return fmt.Errorf("Failed to extract expectations: %v", err)
+				}
+				notes = append(notes, l...)
+			}
+		}
+		e.notes = notes
+	}
+	if e.markers == nil {
+		if err := e.getMarkers(); err != nil {
+			return err
+		}
+	}
+	var err error
+	ms := make(map[string]method, len(methods))
+	for name, f := range methods {
+		mi := method{f: reflect.ValueOf(f)}
+		mi.converters = make([]converter, mi.f.Type().NumIn())
+		for i := 0; i < len(mi.converters); i++ {
+			mi.converters[i], err = e.buildConverter(mi.f.Type().In(i))
+			if err != nil {
+				return fmt.Errorf("invalid method %v: %v", name, err)
+			}
+		}
+		ms[name] = mi
+	}
+	for _, n := range e.notes {
+		mi, ok := ms[n.Name]
+		if !ok {
+			continue
+		}
+		params := make([]reflect.Value, len(mi.converters))
+		args := n.Args
+		for i, convert := range mi.converters {
+			params[i], args, err = convert(n, args)
+			if err != nil {
+				return fmt.Errorf("%v: %v", e.fset.Position(n.Pos), err)
+			}
+		}
+		if len(args) > 0 {
+			return fmt.Errorf("%v: unwanted args got %+v extra", e.fset.Position(n.Pos), args)
+		}
+		//TODO: catch the error returned from the method
+		mi.f.Call(params)
+	}
+	return nil
+}
+
+type marker struct {
+	name  string
+	start token.Pos
+	end   token.Pos
+}
+
+func (e *Exported) getMarkers() error {
+	e.markers = make(map[string]marker)
+	for _, n := range e.notes {
+		var name string
+		var pattern interface{}
+		switch {
+		case n.Args == nil:
+			// simple identifier form
+			name = n.Name
+			pattern = n.Name
+		case n.Name == "mark":
+			if len(n.Args) != 2 {
+				return fmt.Errorf("%v: expected 2 args to mark, got %v", e.fset.Position(n.Pos), len(n.Args))
+			}
+			ident, ok := n.Args[0].(expect.Identifier)
+			if !ok {
+				return fmt.Errorf("%v: expected identifier, got %T", e.fset.Position(n.Pos), n.Args[0])
+			}
+			name = string(ident)
+			pattern = n.Args[1]
+		default:
+			// not a marker note, so skip it
+			continue
+		}
+		if old, found := e.markers[name]; found {
+			return fmt.Errorf("%v: marker %v already exists at %v", e.fset.Position(n.Pos), name, e.fset.Position(old.start))
+		}
+		start, end, err := expect.MatchBefore(e.fset, e.fileContents, n.Pos, pattern)
+		if err != nil {
+			return err
+		}
+		if start == token.NoPos {
+			return fmt.Errorf("%v: pattern %s did not match", e.fset.Position(n.Pos), pattern)
+		}
+		e.markers[name] = marker{
+			name:  name,
+			start: start,
+			end:   end,
+		}
+	}
+	return nil
+}
+
+var (
+	noteType       = reflect.TypeOf((*expect.Note)(nil))
+	identifierType = reflect.TypeOf(expect.Identifier(""))
+	posType        = reflect.TypeOf(token.Pos(0))
+	positionType   = reflect.TypeOf(token.Position{})
+)
+
+// converter converts from a marker's argument parsed from the comment to
+// reflect values passed to the method during Invoke.
+// It takes the args remaining, and returns the args it did not consume.
+// This allows a converter to consume 0 args for well known types, or multiple
+// args for compound types.
+type converter func(*expect.Note, []interface{}) (reflect.Value, []interface{}, error)
+
+// method is used to track information about Invoke methods that is expensive to
+// calculate so that we can work it out once rather than per marker.
+type method struct {
+	f          reflect.Value // the reflect value of the passed in method
+	converters []converter   // the parameter converters for the method
+}
+
+// buildConverter works out what function should be used to go from an ast expressions to a reflect
+// value of the type expected by a method.
+// It is called when only the target type is know, it returns converters that are flexible across
+// all supported expression types for that target type.
+func (e *Exported) buildConverter(pt reflect.Type) (converter, error) {
+	switch {
+	case pt == noteType:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			return reflect.ValueOf(n), args, nil
+		}, nil
+	case pt == posType:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			pos, remains, err := e.posConverter(n, args)
+			if err != nil {
+				return reflect.Value{}, nil, err
+			}
+			return reflect.ValueOf(pos), remains, nil
+		}, nil
+	case pt == positionType:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			pos, remains, err := e.posConverter(n, args)
+			if err != nil {
+				return reflect.Value{}, nil, err
+			}
+			return reflect.ValueOf(e.fset.Position(pos)), remains, nil
+		}, nil
+	case pt == identifierType:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			arg := args[0]
+			args = args[1:]
+			switch arg := arg.(type) {
+			case expect.Identifier:
+				return reflect.ValueOf(arg), args, nil
+			default:
+				return reflect.Value{}, nil, fmt.Errorf("cannot convert %v to string", arg)
+			}
+		}, nil
+	case pt.Kind() == reflect.String:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			arg := args[0]
+			args = args[1:]
+			switch arg := arg.(type) {
+			case expect.Identifier:
+				return reflect.ValueOf(string(arg)), args, nil
+			case string:
+				return reflect.ValueOf(arg), args, nil
+			default:
+				return reflect.Value{}, nil, fmt.Errorf("cannot convert %v to string", arg)
+			}
+		}, nil
+	case pt.Kind() == reflect.Int64:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			arg := args[0]
+			args = args[1:]
+			switch arg := arg.(type) {
+			case int64:
+				return reflect.ValueOf(arg), args, nil
+			default:
+				return reflect.Value{}, nil, fmt.Errorf("cannot convert %v to int", arg)
+			}
+		}, nil
+	case pt.Kind() == reflect.Bool:
+		return func(n *expect.Note, args []interface{}) (reflect.Value, []interface{}, error) {
+			arg := args[0]
+			args = args[1:]
+			b, ok := arg.(bool)
+			if !ok {
+				return reflect.Value{}, nil, fmt.Errorf("cannot convert %v to bool", arg)
+			}
+			return reflect.ValueOf(b), args, nil
+		}, nil
+	default:
+		return nil, fmt.Errorf("param has invalid type %v", pt)
+	}
+}
+
+func (e *Exported) posConverter(n *expect.Note, args []interface{}) (token.Pos, []interface{}, error) {
+	if len(args) < 1 {
+		return 0, nil, fmt.Errorf("missing argument")
+	}
+	arg := args[0]
+	args = args[1:]
+	switch arg := arg.(type) {
+	case expect.Identifier:
+		// look up an marker by name
+		p, ok := e.markers[string(arg)]
+		if !ok {
+			return 0, nil, fmt.Errorf("cannot find marker %v", arg)
+		}
+		return p.start, args, nil
+	case string:
+		p, _, err := expect.MatchBefore(e.fset, e.fileContents, n.Pos, arg)
+		if err != nil {
+			return 0, nil, err
+		}
+		if p == token.NoPos {
+			return 0, nil, fmt.Errorf("%v: pattern %s did not match", e.fset.Position(n.Pos), arg)
+		}
+		return p, args, nil
+	case *regexp.Regexp:
+		p, _, err := expect.MatchBefore(e.fset, e.fileContents, n.Pos, arg)
+		if err != nil {
+			return 0, nil, err
+		}
+		if p == token.NoPos {
+			return 0, nil, fmt.Errorf("%v: pattern %s did not match", e.fset.Position(n.Pos), arg)
+		}
+		return p, args, nil
+	default:
+		return 0, nil, fmt.Errorf("cannot convert %v to pos", arg)
+	}
+}

go/packages/packagestest/expect_test.go

@@ -0,0 +1,51 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packagestest_test
+
+import (
+	"go/token"
+	"testing"
+
+	"golang.org/x/tools/go/expect"
+	"golang.org/x/tools/go/packages/packagestest"
+)
+
+func TestExpect(t *testing.T) {
+	exported := packagestest.Export(t, packagestest.GOPATH, []packagestest.Module{{
+		Name:  "golang.org/fake",
+		Files: packagestest.MustCopyFileTree("testdata"),
+	}})
+	defer exported.Cleanup()
+	count := 0
+	if err := exported.Expect(map[string]interface{}{
+		"check": func(src, target token.Position) {
+			count++
+		},
+		"boolArg": func(n *expect.Note, yes, no bool) {
+			if !yes {
+				t.Errorf("Expected boolArg first param to be true")
+			}
+			if no {
+				t.Errorf("Expected boolArg second param to be false")
+			}
+		},
+		"intArg": func(n *expect.Note, i int64) {
+			if i != 42 {
+				t.Errorf("Expected intarg to be 42")
+			}
+		},
+		"stringArg": func(n *expect.Note, name expect.Identifier, value string) {
+			if string(name) != value {
+				t.Errorf("Got string arg %v expected %v", value, name)
+			}
+		},
+		"directNote": func(n *expect.Note) {},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if count == 0 {
+		t.Fatalf("No tests were run")
+	}
+}

go/packages/packagestest/export.go

@@ -14,6 +14,7 @@ package packagestest
 import (
 	"flag"
 	"fmt"
+	"go/token"
 	"io/ioutil"
 	"log"
 	"os"
@@ -21,6 +22,7 @@ import (
 	"strings"
 	"testing"
 
+	"golang.org/x/tools/go/expect"
 	"golang.org/x/tools/go/packages"
 )
 
@@ -51,9 +53,13 @@ type Exported struct {
 	// Exactly what it will contain varies depending on the Exporter being used.
 	Config *packages.Config
 
-	temp    string
-	primary string
-	written map[string]map[string]string
+	temp     string                       // the temporary directory that was exported to
+	primary  string                       // the first non GOROOT module that was exported
+	written  map[string]map[string]string // the full set of exported files
+	fset     *token.FileSet               // The file set used when parsing expectations
+	notes    []*expect.Note               // The list of expectations extracted from go source files
+	markers  map[string]marker            // The set of markers extracted from go source files
+	contents map[string][]byte
 }
 
 // Exporter implementations are responsible for converting from the generic description of some
@@ -107,6 +113,8 @@ func Export(t *testing.T, exporter Exporter, modules []Module) *Exported {
 		temp:     temp,
 		primary:  modules[0].Name,
 		written:  map[string]map[string]string{},
+		fset:     token.NewFileSet(),
+		contents: map[string][]byte{},
 	}
 	defer func() {
 		if t.Failed() || t.Skipped() {
@@ -201,6 +209,30 @@ func Copy(source string) Writer {
 	}
 }
 
+// MustCopyFileTree returns a file set for a module based on a real directory tree.
+// It scans the directory tree anchored at root and adds a Copy writer to the
+// map for every file found.
+// This is to enable the common case in tests where you have a full copy of the
+// package in your testdata.
+// This will panic if there is any kind of error trying to walk the file tree.
+func MustCopyFileTree(root string) map[string]interface{} {
+	result := map[string]interface{}{}
+	if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+		fragment, err := filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+		result[fragment] = Copy(path)
+		return nil
+	}); err != nil {
+		log.Panic(fmt.Sprintf("MustCopyFileTree failed: %v", err))
+	}
+	return result
+}
+
 // Cleanup removes the temporary directory (unless the --skip-cleanup flag was set)
 // It is safe to call cleanup multiple times.
 func (e *Exported) Cleanup() {
@@ -227,3 +259,14 @@ func (e *Exported) File(module, fragment string) string {
 	}
 	return ""
 }
+
+func (e *Exported) fileContents(filename string) ([]byte, error) {
+	if content, found := e.contents[filename]; found {
+		return content, nil
+	}
+	content, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return content, nil
+}

go/packages/packagestest/testdata/test.go

@@ -0,0 +1,20 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fake1
+
+// This is a test file for the behaviors in Exported.Expect.
+
+type AThing string //@AThing,mark(StringThing, "AThing"),mark(REThing,re`.T.*g`)
+
+type Match string //@check("Match",re`[[:upper:]]`)
+
+//@check(AThing, StringThing)
+//@check(AThing, REThing)
+
+//@boolArg(true, false)
+//@intArg(42)
+//@stringArg(PlainString, "PlainString")
+//@stringArg(IdentAsString,IdentAsString)
+//@directNote()