go/parser: report //go:build-derived Go version in ast.File.GoVersion

For #57001, compilers and others tools will need to understand that
a different Go version can be used in different files in a program,
according to the //go:build lines in those files.

Update go/parser to populate the new ast.File.GoVersion field.

This requires running the go/scanner in ParseComments mode
always and then implementing discarding of comments in the
parser instead of the scanner. The same work is done either way,
since the scanner was already preparing the comment result
and then looping. The loop has just moved into go/parser.

Also make the same changes to cmd/compile/internal/syntax,
both because they're necessary and to keep in sync with go/parser.

For #59033.

Change-Id: I7b867f5f9aaaccdca94af146b061d16d9a3fd07f
Reviewed-on: https://go-review.googlesource.com/c/go/+/476277
Auto-Submit: Russ Cox <rsc@golang.org>
Reviewed-by: Robert Griesemer <gri@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Russ Cox <rsc@golang.org>

src/cmd/compile/internal/syntax/nodes.go

@@ -34,10 +34,11 @@ func (*node) aNode()     {}
 
 // package PkgName; DeclList[0], DeclList[1], ...
 type File struct {
-	Pragma   Pragma
-	PkgName  *Name
-	DeclList []Decl
-	EOF      Pos
+	Pragma    Pragma
+	PkgName   *Name
+	DeclList  []Decl
+	EOF       Pos
+	GoVersion string
 	node
 }
 

src/cmd/compile/internal/syntax/parser.go

@@ -6,6 +6,7 @@ package syntax
 
 import (
 	"fmt"
+	"go/build/constraint"
 	"io"
 	"strconv"
 	"strings"
@@ -21,17 +22,20 @@ type parser struct {
 	pragh PragmaHandler
 	scanner
 
-	base   *PosBase // current position base
-	first  error    // first error encountered
-	errcnt int      // number of errors encountered
-	pragma Pragma   // pragmas
+	base      *PosBase // current position base
+	first     error    // first error encountered
+	errcnt    int      // number of errors encountered
+	pragma    Pragma   // pragmas
+	goVersion string   // Go version from //go:build line
 
+	top    bool   // in top of file (before package clause)
 	fnest  int    // function nesting level (for error handling)
 	xnest  int    // expression nesting level (for complit ambiguity resolution)
 	indent []byte // tracing support
 }
 
 func (p *parser) init(file *PosBase, r io.Reader, errh ErrorHandler, pragh PragmaHandler, mode Mode) {
+	p.top = true
 	p.file = file
 	p.errh = errh
 	p.mode = mode
@@ -70,8 +74,15 @@ func (p *parser) init(file *PosBase, r io.Reader, errh ErrorHandler, pragh Pragm
 			}
 
 			// go: directive (but be conservative and test)
-			if pragh != nil && strings.HasPrefix(text, "go:") {
-				p.pragma = pragh(p.posAt(line, col+2), p.scanner.blank, text, p.pragma) // +2 to skip over // or /*
+			if strings.HasPrefix(text, "go:") {
+				if p.top && strings.HasPrefix(msg, "//go:build") {
+					if x, err := constraint.Parse(msg); err == nil {
+						p.goVersion = constraint.GoVersion(x)
+					}
+				}
+				if pragh != nil {
+					p.pragma = pragh(p.posAt(line, col+2), p.scanner.blank, text, p.pragma) // +2 to skip over // or /*
+				}
 			}
 		},
 		directives,
@@ -388,6 +399,8 @@ func (p *parser) fileOrNil() *File {
 	f.pos = p.pos()
 
 	// PackageClause
+	f.GoVersion = p.goVersion
+	p.top = false
 	if !p.got(_Package) {
 		p.syntaxError("package statement must be first")
 		return nil

src/go/build/deps_test.go

@@ -263,13 +263,15 @@ var depsRules = `
 	< go/token
 	< go/scanner
 	< go/ast
-	< go/internal/typeparams
-	< go/parser;
+	< go/internal/typeparams;
 
 	FMT
 	< go/build/constraint, go/doc/comment;
 
-	go/build/constraint, go/doc/comment, go/parser, text/tabwriter
+	go/internal/typeparams, go/build/constraint
+	< go/parser;
+
+	go/doc/comment, go/parser, text/tabwriter
 	< go/printer
 	< go/format;
 

src/go/parser/parser.go

@@ -18,9 +18,11 @@ package parser
 import (
 	"fmt"
 	"go/ast"
+	"go/build/constraint"
 	"go/internal/typeparams"
 	"go/scanner"
 	"go/token"
+	"strings"
 )
 
 // The parser structure holds the parser's internal state.
@@ -38,6 +40,8 @@ type parser struct {
 	comments    []*ast.CommentGroup
 	leadComment *ast.CommentGroup // last lead comment
 	lineComment *ast.CommentGroup // last line comment
+	top         bool              // in top of file (before package clause)
+	goVersion   string            // minimum Go version found in //go:build comment
 
 	// Next token
 	pos token.Pos   // token position
@@ -64,13 +68,10 @@ type parser struct {
 
 func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode Mode) {
 	p.file = fset.AddFile(filename, -1, len(src))
-	var m scanner.Mode
-	if mode&ParseComments != 0 {
-		m = scanner.ScanComments
-	}
 	eh := func(pos token.Position, msg string) { p.errors.Add(pos, msg) }
-	p.scanner.Init(p.file, src, eh, m)
+	p.scanner.Init(p.file, src, eh, scanner.ScanComments)
 
+	p.top = true
 	p.mode = mode
 	p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently)
 	p.next()
@@ -142,7 +143,23 @@ func (p *parser) next0() {
 		}
 	}
 
-	p.pos, p.tok, p.lit = p.scanner.Scan()
+	for {
+		p.pos, p.tok, p.lit = p.scanner.Scan()
+		if p.tok == token.COMMENT {
+			if p.top && strings.HasPrefix(p.lit, "//go:build") {
+				if x, err := constraint.Parse(p.lit); err == nil {
+					p.goVersion = constraint.GoVersion(x)
+				}
+			}
+			if p.mode&ParseComments == 0 {
+				continue
+			}
+		} else {
+			// Found a non-comment; top of file is over.
+			p.top = false
+		}
+		break
+	}
 }
 
 // Consume a comment and return it and the line on which it ends.
@@ -2851,6 +2868,7 @@ func (p *parser) parseFile() *ast.File {
 		FileEnd:   token.Pos(p.file.Base() + p.file.Size()),
 		Imports:   p.imports,
 		Comments:  p.comments,
+		GoVersion: p.goVersion,
 	}
 	var declErr func(token.Pos, string)
 	if p.mode&DeclarationErrors != 0 {

src/go/parser/parser_test.go

@@ -780,3 +780,23 @@ func TestIssue59180(t *testing.T) {
 		}
 	}
 }
+
+func TestGoVersion(t *testing.T) {
+	fset := token.NewFileSet()
+	pkgs, err := ParseDir(fset, "./testdata/goversion", nil, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, p := range pkgs {
+		want := strings.ReplaceAll(p.Name, "_", ".")
+		if want == "none" {
+			want = ""
+		}
+		for _, f := range p.Files {
+			if f.GoVersion != want {
+				t.Errorf("%s: GoVersion = %q, want %q", fset.Position(f.Pos()), f.GoVersion, want)
+			}
+		}
+	}
+}

src/go/parser/testdata/goversion/t01.go

@@ -0,0 +1,3 @@
+//go:build windows
+
+package none

src/go/parser/testdata/goversion/t02.go

@@ -0,0 +1,3 @@
+//go:build linux && go1.2
+
+package go1_2

src/go/parser/testdata/goversion/t03.go

@@ -0,0 +1,3 @@
+//go:build linux && go1.2 || windows
+
+package none

src/go/parser/testdata/goversion/t04.go

@@ -0,0 +1,5 @@
+// copyright notice
+
+//go:build (linux && go1.2) || (windows && go1.1)
+
+package go1_1

src/go/parser/testdata/goversion/t05.go

@@ -0,0 +1,3 @@
+//go:build linux && go1.2 && go1.4
+
+package go1_4

src/go/parser/testdata/goversion/t06.go

@@ -0,0 +1,3 @@
+//go:build go1
+
+package go1