cmd/compile: skip substituting closures in unsafe builtins arguments

For unsafe.{Alignof,Offsetof,Sizeof}, subster will transform them them
to OLITERAL nodes, and discard their arguments. However, any closure in
their children nodes were already processed and added to declaration
queue. Thus, we lack of information for generating instantiation for
the closure.

To fix it, just skip substituting the closures if we are going to edit
the children nodes of unsafe builtins.

Fixes #53390

Change-Id: Ia815cd05af9dc0491f10faac4399f378ac53dec6
Reviewed-on: https://go-review.googlesource.com/c/go/+/412794
Reviewed-by: Keith Randall <khr@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Keith Randall <khr@google.com>
Run-TryBot: Cuong Manh Le <cuong.manhle.vn@gmail.com>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>

src/cmd/compile/internal/noder/stencil.go

@@ -726,6 +726,7 @@ type subster struct {
 	newf        *ir.Func // Func node for the new stenciled function
 	ts          typecheck.Tsubster
 	info        *instInfo // Place to put extra info in the instantiation
+	skipClosure bool      // Skip substituting closures
 
 	// Map from non-nil, non-ONAME node n to slice of all m, where m.Defn = n
 	defnMap map[ir.Node][]**ir.Name
@@ -978,7 +979,20 @@ func (subst *subster) node(n ir.Node) ir.Node {
 			}
 		}
 
+		old := subst.skipClosure
+		// For unsafe.{Alignof,Offsetof,Sizeof}, subster will transform them to OLITERAL nodes,
+		// and discard their arguments. However, their children nodes were already process before,
+		// thus if they contain any closure, the closure was still be added to package declarations
+		// queue for processing later. Thus, genInst will fail to generate instantiation for the
+		// closure because of lacking dictionary information, see issue #53390.
+		if call, ok := m.(*ir.CallExpr); ok && call.X.Op() == ir.ONAME {
+			switch call.X.Name().BuiltinOp {
+			case ir.OALIGNOF, ir.OOFFSETOF, ir.OSIZEOF:
+				subst.skipClosure = true
+			}
+		}
 		ir.EditChildren(m, edit)
+		subst.skipClosure = old
 
 		m.SetTypecheck(1)
 
@@ -1123,6 +1137,9 @@ func (subst *subster) node(n ir.Node) ir.Node {
 			}
 
 		case ir.OCLOSURE:
+			if subst.skipClosure {
+				break
+			}
 			// We're going to create a new closure from scratch, so clear m
 			// to avoid using the ir.Copy by accident until we reassign it.
 			m = nil

test/typeparam/issue53390.go

@@ -0,0 +1,20 @@
+// compile
+
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package p
+
+import "unsafe"
+
+func F[T any](v T) uintptr {
+	return unsafe.Alignof(func() T {
+		func(any) {}(struct{ _ T }{})
+		return v
+	}())
+}
+
+func f() {
+	F(0)
+}