[dev.typeparams] cmd/compile/internal/types2: avoid endless recursion in Comparable predicate

Use a map to detect recursive types.
With this we can now typecheck fixedbugs/issue8501.go.

Updates #43088.

Change-Id: I7fad6ccf6c94268473ff72b09a3158e13a7f4cc3
Reviewed-on: https://go-review.googlesource.com/c/go/+/276374
Trust: Robert Griesemer <gri@golang.org>
Run-TryBot: Robert Griesemer <gri@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Robert Findley <rfindley@google.com>

src/cmd/compile/internal/types2/issues_test.go

@@ -531,3 +531,28 @@ func TestIssue34921(t *testing.T) {
 		pkg = res // res is imported by the next package in this test
 	}
 }
+
+func TestIssue43088(t *testing.T) {
+	// type T1 struct {
+	//         x T2
+	// }
+	//
+	// type T2 struct {
+	//         x struct {
+	//                 x T2
+	//         }
+	// }
+	n1 := NewTypeName(syntax.Pos{}, nil, "T1", nil)
+	T1 := NewNamed(n1, nil, nil)
+	n2 := NewTypeName(syntax.Pos{}, nil, "T2", nil)
+	T2 := NewNamed(n2, nil, nil)
+	s1 := NewStruct([]*Var{NewField(syntax.Pos{}, nil, "x", T2, false)}, nil)
+	T1.SetUnderlying(s1)
+	s2 := NewStruct([]*Var{NewField(syntax.Pos{}, nil, "x", T2, false)}, nil)
+	s3 := NewStruct([]*Var{NewField(syntax.Pos{}, nil, "x", s2, false)}, nil)
+	T2.SetUnderlying(s3)
+
+	// These calls must terminate (no endless recursion).
+	Comparable(T1)
+	Comparable(T2)
+}

src/cmd/compile/internal/types2/predicates.go

@@ -87,6 +87,11 @@ func IsInterface(typ Type) bool {
 
 // Comparable reports whether values of type T are comparable.
 func Comparable(T Type) bool {
+	return comparable(T, nil)
+}
+
+// comparable should only be called by Comparable.
+func comparable(T Type, seen map[Type]bool) bool {
 	// If T is a type parameter not constraint by any type
 	// list (i.e., it's underlying type is the top type),
 	// T is comparable if it has the == method. Otherwise,
@@ -99,6 +104,14 @@ func Comparable(T Type) bool {
 		return t.Bound().IsComparable()
 	}
 
+	if seen[T] {
+		return true
+	}
+	if seen == nil {
+		seen = make(map[Type]bool)
+	}
+	seen[T] = true
+
 	switch t := optype(T.Under()).(type) {
 	case *Basic:
 		// assume invalid types to be comparable
@@ -108,13 +121,13 @@ func Comparable(T Type) bool {
 		return true
 	case *Struct:
 		for _, f := range t.fields {
-			if !Comparable(f.typ) {
+			if !comparable(f.typ, seen) {
 				return false
 			}
 		}
 		return true
 	case *Array:
-		return Comparable(t.elem)
+		return comparable(t.elem, seen)
 	case *Sum:
 		return t.is(Comparable)
 	case *TypeParam:

test/fixedbugs/issue8507.go

@@ -9,7 +9,7 @@
 
 package p
 
-type T struct{ T } // ERROR "invalid recursive type T"
+type T struct{ T } // ERROR "invalid recursive type T|cycle"
 
 func f() {
 	println(T{} == T{})

test/run.go

@@ -2130,6 +2130,4 @@ var excluded = map[string]bool{
 	"fixedbugs/issue7525e.go":  true,
 	"fixedbugs/issue7742.go":   true, // type-checking doesn't terminate
 	"fixedbugs/issue7746.go":   true, // type-checking doesn't terminate
-	"fixedbugs/issue8501.go":   true, // crashes
-	"fixedbugs/issue8507.go":   true, // crashes
 }