cmd/compile,runtime: open code unsafe.Slice

So prevent heavy runtime call overhead, and the compiler will have a
chance to optimize the bound check.

With this optimization, changing runtime/stack.go to use unsafe.Slice
no longer negatively impacts stack copying performance:

name                   old time/op    new time/op    delta
StackCopyWithStkobj-8    16.3ms ± 6%    16.5ms ± 5%   ~     (p=0.382 n=8+8)

name                   old alloc/op   new alloc/op   delta
StackCopyWithStkobj-8     17.0B ± 0%     17.0B ± 0%   ~     (all equal)

name                   old allocs/op  new allocs/op  delta
StackCopyWithStkobj-8      1.00 ± 0%      1.00 ± 0%   ~     (all equal)

Fixes #48798

Change-Id: I731a9a4abd6dd6846f44eece7f86025b7bb1141b
Reviewed-on: https://go-review.googlesource.com/c/go/+/362934
Reviewed-by: Keith Randall <khr@golang.org>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>
Reviewed-by: Keith Randall <khr@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Cuong Manh Le <cuong.manhle.vn@gmail.com>

src/cmd/compile/internal/ssagen/ssa.go

@@ -3921,6 +3921,7 @@ func InitTables() {
 			return s.newValue2(ssa.OpMul64uover, types.NewTuple(types.Types[types.TUINT], types.Types[types.TUINT]), args[0], args[1])
 		},
 		sys.AMD64, sys.I386, sys.MIPS64, sys.RISCV64)
+	alias("runtime", "mulUintptr", "runtime/internal/math", "MulUintptr", all...)
 	add("runtime", "KeepAlive",
 		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
 			data := s.newValue1(ssa.OpIData, s.f.Config.Types.BytePtr, args[0])

src/cmd/compile/internal/typecheck/builtin.go

@@ -134,9 +134,10 @@ var runtimeDecls = [...]struct {
 	{"makeslice64", funcTag, 114},
 	{"makeslicecopy", funcTag, 115},
 	{"growslice", funcTag, 117},
-	{"unsafeslice", funcTag, 118},
+	{"unsafeslicecheckptr", funcTag, 118},
-	{"unsafeslice64", funcTag, 119},
+	{"panicunsafeslicelen", funcTag, 9},
-	{"unsafeslicecheckptr", funcTag, 119},
+	{"panicunsafeslicenilptr", funcTag, 9},
+	{"mulUintptr", funcTag, 119},
 	{"memmove", funcTag, 120},
 	{"memclrNoHeapPointers", funcTag, 121},
 	{"memclrHasPointers", funcTag, 121},
@@ -346,8 +347,8 @@ func runtimeTypes() []*types.Type {
 	typs[115] = newSig(params(typs[1], typs[15], typs[15], typs[7]), params(typs[7]))
 	typs[116] = types.NewSlice(typs[2])
 	typs[117] = newSig(params(typs[1], typs[116], typs[15]), params(typs[116]))
-	typs[118] = newSig(params(typs[1], typs[7], typs[15]), nil)
+	typs[118] = newSig(params(typs[1], typs[7], typs[22]), nil)
-	typs[119] = newSig(params(typs[1], typs[7], typs[22]), nil)
+	typs[119] = newSig(params(typs[5], typs[5]), params(typs[5], typs[6]))
 	typs[120] = newSig(params(typs[3], typs[3], typs[5]), nil)
 	typs[121] = newSig(params(typs[7], typs[5]), nil)
 	typs[122] = newSig(params(typs[3], typs[3], typs[5]), params(typs[6]))

src/cmd/compile/internal/typecheck/builtin/runtime.go

@@ -180,9 +180,11 @@ func makeslice(typ *byte, len int, cap int) unsafe.Pointer
 func makeslice64(typ *byte, len int64, cap int64) unsafe.Pointer
 func makeslicecopy(typ *byte, tolen int, fromlen int, from unsafe.Pointer) unsafe.Pointer
 func growslice(typ *byte, old []any, cap int) (ary []any)
-func unsafeslice(typ *byte, ptr unsafe.Pointer, len int)
-func unsafeslice64(typ *byte, ptr unsafe.Pointer, len int64)
 func unsafeslicecheckptr(typ *byte, ptr unsafe.Pointer, len int64)
+func panicunsafeslicelen()
+func panicunsafeslicenilptr()
+
+func mulUintptr(x, y uintptr) (uintptr, bool)
 
 func memmove(to *any, frm *any, length uintptr)
 func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)

src/cmd/compile/internal/walk/builtin.go

@@ -645,28 +645,67 @@ func walkRecoverFP(nn *ir.CallExpr, init *ir.Nodes) ir.Node {
 func walkUnsafeSlice(n *ir.BinaryExpr, init *ir.Nodes) ir.Node {
 	ptr := safeExpr(n.X, init)
 	len := safeExpr(n.Y, init)
+	sliceType := n.Type()
 
-	fnname := "unsafeslice64"
 	lenType := types.Types[types.TINT64]
+	unsafePtr := typecheck.Conv(ptr, types.Types[types.TUNSAFEPTR])
 
+	// If checkptr enabled, call runtime.unsafeslicecheckptr to check ptr and len.
+	// for simplicity, unsafeslicecheckptr always uses int64.
 	// Type checking guarantees that TIDEAL len/cap are positive and fit in an int.
 	// The case of len or cap overflow when converting TUINT or TUINTPTR to TINT
 	// will be handled by the negative range checks in unsafeslice during runtime.
 	if ir.ShouldCheckPtr(ir.CurFunc, 1) {
-		fnname = "unsafeslicecheckptr"
+		fnname := "unsafeslicecheckptr"
-		// for simplicity, unsafeslicecheckptr always uses int64
+		fn := typecheck.LookupRuntime(fnname)
-	} else if len.Type().IsKind(types.TIDEAL) || len.Type().Size() <= types.Types[types.TUINT].Size() {
+		init.Append(mkcall1(fn, nil, init, reflectdata.TypePtr(sliceType.Elem()), unsafePtr, typecheck.Conv(len, lenType)))
-		fnname = "unsafeslice"
+	} else {
-		lenType = types.Types[types.TINT]
+		// Otherwise, open code unsafe.Slice to prevent runtime call overhead.
+		// Keep this code in sync with runtime.unsafeslice{,64}
+		if len.Type().IsKind(types.TIDEAL) || len.Type().Size() <= types.Types[types.TUINT].Size() {
+			lenType = types.Types[types.TINT]
+		} else {
+			// len64 := int64(len)
+			// if int64(int(len64)) != len64 {
+			//     panicunsafeslicelen()
+			// }
+			len64 := typecheck.Conv(len, lenType)
+			nif := ir.NewIfStmt(base.Pos, nil, nil, nil)
+			nif.Cond = ir.NewBinaryExpr(base.Pos, ir.ONE, typecheck.Conv(typecheck.Conv(len64, types.Types[types.TINT]), lenType), len64)
+			nif.Body.Append(mkcall("panicunsafeslicelen", nil, &nif.Body))
+			appendWalkStmt(init, nif)
+		}
+
+		// if len < 0 { panicunsafeslicelen() }
+		nif := ir.NewIfStmt(base.Pos, nil, nil, nil)
+		nif.Cond = ir.NewBinaryExpr(base.Pos, ir.OLT, typecheck.Conv(len, lenType), ir.NewInt(0))
+		nif.Body.Append(mkcall("panicunsafeslicelen", nil, &nif.Body))
+		appendWalkStmt(init, nif)
+
+		// mem, overflow := runtime.mulUintptr(et.size, len)
+		mem := typecheck.Temp(types.Types[types.TUINTPTR])
+		overflow := typecheck.Temp(types.Types[types.TBOOL])
+		fn := typecheck.LookupRuntime("mulUintptr")
+		call := mkcall1(fn, fn.Type().Results(), init, ir.NewInt(sliceType.Elem().Size()), typecheck.Conv(typecheck.Conv(len, lenType), types.Types[types.TUINTPTR]))
+		appendWalkStmt(init, ir.NewAssignListStmt(base.Pos, ir.OAS2, []ir.Node{mem, overflow}, []ir.Node{call}))
+
+		// if overflow || mem > -uintptr(ptr) {
+		//     if ptr == nil {
+		//         panicunsafesliceptrnil()
+		//     }
+		//     panicunsafeslicelen()
+		// }
+		nif = ir.NewIfStmt(base.Pos, nil, nil, nil)
+		memCond := ir.NewBinaryExpr(base.Pos, ir.OGT, mem, ir.NewUnaryExpr(base.Pos, ir.ONEG, typecheck.Conv(unsafePtr, types.Types[types.TUINTPTR])))
+		nif.Cond = ir.NewLogicalExpr(base.Pos, ir.OOROR, overflow, memCond)
+		nifPtr := ir.NewIfStmt(base.Pos, nil, nil, nil)
+		nifPtr.Cond = ir.NewBinaryExpr(base.Pos, ir.OEQ, unsafePtr, typecheck.NodNil())
+		nifPtr.Body.Append(mkcall("panicunsafeslicenilptr", nil, &nifPtr.Body))
+		nif.Body.Append(nifPtr, mkcall("panicunsafeslicelen", nil, &nif.Body))
+		appendWalkStmt(init, nif)
 	}
 
-	t := n.Type()
+	h := ir.NewSliceHeaderExpr(n.Pos(), sliceType,
-
-	// Call runtime.unsafeslice{,64,checkptr} to check ptr and len.
-	fn := typecheck.LookupRuntime(fnname)
-	init.Append(mkcall1(fn, nil, init, reflectdata.TypePtr(t.Elem()), typecheck.Conv(ptr, types.Types[types.TUNSAFEPTR]), typecheck.Conv(len, lenType)))
-
-	h := ir.NewSliceHeaderExpr(n.Pos(), t,
 		typecheck.Conv(ptr, types.Types[types.TUNSAFEPTR]),
 		typecheck.Conv(len, types.Types[types.TINT]),
 		typecheck.Conv(len, types.Types[types.TINT]))

src/runtime/slice.go

@@ -117,6 +117,13 @@ func makeslice64(et *_type, len64, cap64 int64) unsafe.Pointer {
 	return makeslice(et, len, cap)
 }
 
+// This is a wrapper over runtime/internal/math.MulUintptr,
+// so the compiler can recognize and treat it as an intrinsic.
+func mulUintptr(a, b uintptr) (uintptr, bool) {
+	return math.MulUintptr(a, b)
+}
+
+// Keep this code in sync with cmd/compile/internal/walk/builtin.go:walkUnsafeSlice
 func unsafeslice(et *_type, ptr unsafe.Pointer, len int) {
 	if len < 0 {
 		panicunsafeslicelen()
@@ -125,12 +132,13 @@ func unsafeslice(et *_type, ptr unsafe.Pointer, len int) {
 	mem, overflow := math.MulUintptr(et.size, uintptr(len))
 	if overflow || mem > -uintptr(ptr) {
 		if ptr == nil {
-			panic(errorString("unsafe.Slice: ptr is nil and len is not zero"))
+			panicunsafeslicenilptr()
 		}
 		panicunsafeslicelen()
 	}
 }
 
+// Keep this code in sync with cmd/compile/internal/walk/builtin.go:walkUnsafeSlice
 func unsafeslice64(et *_type, ptr unsafe.Pointer, len64 int64) {
 	len := int(len64)
 	if int64(len) != len64 {
@@ -153,6 +161,10 @@ func panicunsafeslicelen() {
 	panic(errorString("unsafe.Slice: len out of range"))
 }
 
+func panicunsafeslicenilptr() {
+	panic(errorString("unsafe.Slice: ptr is nil and len is not zero"))
+}
+
 // growslice handles slice growth during append.
 // It is passed the slice element type, the old slice, and the desired new minimum capacity,
 // and it returns a new slice with at least that capacity, with the old data