cmd/link: mangle certain instantiated function name in plugin mode

In plugin mode, we mangle the type symbol name so it doesn't
contain characters that may confuse the external linker. With
generics, instantiated function name includes type names, so it
may also contain such characters and so also needs to be mangled.

Fixes #58800.

Change-Id: Ibb08c95b89b8a815ccef98193d3a025e9d4756cc
Reviewed-on: https://go-review.googlesource.com/c/go/+/500095
Reviewed-by: Than McIntosh <thanm@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Cherry Mui <cherryyz@google.com>

src/cmd/cgo/internal/testplugin/plugin_test.go

@@ -388,3 +388,10 @@ func TestForkExec(t *testing.T) {
 		t.Fatalf("subprocess hang")
 	}
 }
+
+func TestGeneric(t *testing.T) {
+	// Issue 58800: generic function name may contain weird characters
+	// that confuse the external linker.
+	globalSkip(t)
+	goCmd(t, "build", "-buildmode=plugin", "-o", "generic.so", "./generic/plugin.go")
+}

src/cmd/cgo/internal/testplugin/testdata/generic/plugin.go

@@ -0,0 +1,22 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Instantiated function name may contain weird characters
+// that confuse the external linker, so it needs to be
+// mangled.
+
+package main
+
+//go:noinline
+func F[T any]() {}
+
+type S struct {
+	X int `parser:"|@@)"`
+}
+
+func P() {
+	F[S]()
+}
+
+func main() {}

src/cmd/link/internal/ld/lib.go

@@ -984,7 +984,9 @@ func (ctxt *Link) mangleTypeSym() {
 // Leave type:runtime. symbols alone, because other parts of
 // the linker manipulates them.
 func typeSymbolMangle(name string) string {
-	if !strings.HasPrefix(name, "type:") {
+	isType := strings.HasPrefix(name, "type:")
+	if !isType && !strings.Contains(name, "@") {
+		// Issue 58800: instantiated symbols may include a type name, which may contain "@"
 		return name
 	}
 	if strings.HasPrefix(name, "type:runtime.") {
@@ -993,12 +995,22 @@ func typeSymbolMangle(name string) string {
 	if len(name) <= 14 && !strings.Contains(name, "@") { // Issue 19529
 		return name
 	}
-	hash := notsha256.Sum256([]byte(name))
-	prefix := "type:"
-	if name[5] == '.' {
-		prefix = "type:."
+	if isType {
+		hash := notsha256.Sum256([]byte(name[5:]))
+		prefix := "type:"
+		if name[5] == '.' {
+			prefix = "type:."
+		}
+		return prefix + base64.StdEncoding.EncodeToString(hash[:6])
 	}
-	return prefix + base64.StdEncoding.EncodeToString(hash[:6])
+	// instantiated symbol, replace type name in []
+	i := strings.IndexByte(name, '[')
+	j := strings.LastIndexByte(name, ']')
+	if j == -1 {
+		j = len(name)
+	}
+	hash := notsha256.Sum256([]byte(name[i+1 : j]))
+	return name[:i+1] + base64.StdEncoding.EncodeToString(hash[:6]) + name[j:]
 }
 
 /*