diff --git a/src/cmd/compile/internal/ssagen/ssa.go b/src/cmd/compile/internal/ssagen/ssa.go index 686547f175..f8ed8765f9 100644 --- a/src/cmd/compile/internal/ssagen/ssa.go +++ b/src/cmd/compile/internal/ssagen/ssa.go @@ -3921,6 +3921,7 @@ func InitTables() { return s.newValue2(ssa.OpMul64uover, types.NewTuple(types.Types[types.TUINT], types.Types[types.TUINT]), args[0], args[1]) }, sys.AMD64, sys.I386, sys.MIPS64, sys.RISCV64) + alias("runtime", "mulUintptr", "runtime/internal/math", "MulUintptr", all...) add("runtime", "KeepAlive", func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { data := s.newValue1(ssa.OpIData, s.f.Config.Types.BytePtr, args[0]) diff --git a/src/cmd/compile/internal/typecheck/builtin.go b/src/cmd/compile/internal/typecheck/builtin.go index 581928c005..875af37215 100644 --- a/src/cmd/compile/internal/typecheck/builtin.go +++ b/src/cmd/compile/internal/typecheck/builtin.go @@ -134,9 +134,10 @@ var runtimeDecls = [...]struct { {"makeslice64", funcTag, 114}, {"makeslicecopy", funcTag, 115}, {"growslice", funcTag, 117}, - {"unsafeslice", funcTag, 118}, - {"unsafeslice64", funcTag, 119}, - {"unsafeslicecheckptr", funcTag, 119}, + {"unsafeslicecheckptr", funcTag, 118}, + {"panicunsafeslicelen", funcTag, 9}, + {"panicunsafeslicenilptr", funcTag, 9}, + {"mulUintptr", funcTag, 119}, {"memmove", funcTag, 120}, {"memclrNoHeapPointers", funcTag, 121}, {"memclrHasPointers", funcTag, 121}, @@ -346,8 +347,8 @@ func runtimeTypes() []*types.Type { typs[115] = newSig(params(typs[1], typs[15], typs[15], typs[7]), params(typs[7])) typs[116] = types.NewSlice(typs[2]) typs[117] = newSig(params(typs[1], typs[116], typs[15]), params(typs[116])) - typs[118] = newSig(params(typs[1], typs[7], typs[15]), nil) - typs[119] = newSig(params(typs[1], typs[7], typs[22]), nil) + typs[118] = newSig(params(typs[1], typs[7], typs[22]), nil) + typs[119] = newSig(params(typs[5], typs[5]), params(typs[5], typs[6])) typs[120] = newSig(params(typs[3], typs[3], typs[5]), nil) typs[121] = newSig(params(typs[7], typs[5]), nil) typs[122] = newSig(params(typs[3], typs[3], typs[5]), params(typs[6])) diff --git a/src/cmd/compile/internal/typecheck/builtin/runtime.go b/src/cmd/compile/internal/typecheck/builtin/runtime.go index 04ae4f23a3..dd19eefa29 100644 --- a/src/cmd/compile/internal/typecheck/builtin/runtime.go +++ b/src/cmd/compile/internal/typecheck/builtin/runtime.go @@ -180,9 +180,11 @@ func makeslice(typ *byte, len int, cap int) unsafe.Pointer func makeslice64(typ *byte, len int64, cap int64) unsafe.Pointer func makeslicecopy(typ *byte, tolen int, fromlen int, from unsafe.Pointer) unsafe.Pointer func growslice(typ *byte, old []any, cap int) (ary []any) -func unsafeslice(typ *byte, ptr unsafe.Pointer, len int) -func unsafeslice64(typ *byte, ptr unsafe.Pointer, len int64) func unsafeslicecheckptr(typ *byte, ptr unsafe.Pointer, len int64) +func panicunsafeslicelen() +func panicunsafeslicenilptr() + +func mulUintptr(x, y uintptr) (uintptr, bool) func memmove(to *any, frm *any, length uintptr) func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) diff --git a/src/cmd/compile/internal/walk/builtin.go b/src/cmd/compile/internal/walk/builtin.go index d7b553ed0c..a11031b3d0 100644 --- a/src/cmd/compile/internal/walk/builtin.go +++ b/src/cmd/compile/internal/walk/builtin.go @@ -645,28 +645,67 @@ func walkRecoverFP(nn *ir.CallExpr, init *ir.Nodes) ir.Node { func walkUnsafeSlice(n *ir.BinaryExpr, init *ir.Nodes) ir.Node { ptr := safeExpr(n.X, init) len := safeExpr(n.Y, init) + sliceType := n.Type() - fnname := "unsafeslice64" lenType := types.Types[types.TINT64] + unsafePtr := typecheck.Conv(ptr, types.Types[types.TUNSAFEPTR]) + // If checkptr enabled, call runtime.unsafeslicecheckptr to check ptr and len. + // for simplicity, unsafeslicecheckptr always uses int64. // Type checking guarantees that TIDEAL len/cap are positive and fit in an int. // The case of len or cap overflow when converting TUINT or TUINTPTR to TINT // will be handled by the negative range checks in unsafeslice during runtime. if ir.ShouldCheckPtr(ir.CurFunc, 1) { - fnname = "unsafeslicecheckptr" - // for simplicity, unsafeslicecheckptr always uses int64 - } else if len.Type().IsKind(types.TIDEAL) || len.Type().Size() <= types.Types[types.TUINT].Size() { - fnname = "unsafeslice" - lenType = types.Types[types.TINT] + fnname := "unsafeslicecheckptr" + fn := typecheck.LookupRuntime(fnname) + init.Append(mkcall1(fn, nil, init, reflectdata.TypePtr(sliceType.Elem()), unsafePtr, typecheck.Conv(len, lenType))) + } else { + // Otherwise, open code unsafe.Slice to prevent runtime call overhead. + // Keep this code in sync with runtime.unsafeslice{,64} + if len.Type().IsKind(types.TIDEAL) || len.Type().Size() <= types.Types[types.TUINT].Size() { + lenType = types.Types[types.TINT] + } else { + // len64 := int64(len) + // if int64(int(len64)) != len64 { + // panicunsafeslicelen() + // } + len64 := typecheck.Conv(len, lenType) + nif := ir.NewIfStmt(base.Pos, nil, nil, nil) + nif.Cond = ir.NewBinaryExpr(base.Pos, ir.ONE, typecheck.Conv(typecheck.Conv(len64, types.Types[types.TINT]), lenType), len64) + nif.Body.Append(mkcall("panicunsafeslicelen", nil, &nif.Body)) + appendWalkStmt(init, nif) + } + + // if len < 0 { panicunsafeslicelen() } + nif := ir.NewIfStmt(base.Pos, nil, nil, nil) + nif.Cond = ir.NewBinaryExpr(base.Pos, ir.OLT, typecheck.Conv(len, lenType), ir.NewInt(0)) + nif.Body.Append(mkcall("panicunsafeslicelen", nil, &nif.Body)) + appendWalkStmt(init, nif) + + // mem, overflow := runtime.mulUintptr(et.size, len) + mem := typecheck.Temp(types.Types[types.TUINTPTR]) + overflow := typecheck.Temp(types.Types[types.TBOOL]) + fn := typecheck.LookupRuntime("mulUintptr") + call := mkcall1(fn, fn.Type().Results(), init, ir.NewInt(sliceType.Elem().Size()), typecheck.Conv(typecheck.Conv(len, lenType), types.Types[types.TUINTPTR])) + appendWalkStmt(init, ir.NewAssignListStmt(base.Pos, ir.OAS2, []ir.Node{mem, overflow}, []ir.Node{call})) + + // if overflow || mem > -uintptr(ptr) { + // if ptr == nil { + // panicunsafesliceptrnil() + // } + // panicunsafeslicelen() + // } + nif = ir.NewIfStmt(base.Pos, nil, nil, nil) + memCond := ir.NewBinaryExpr(base.Pos, ir.OGT, mem, ir.NewUnaryExpr(base.Pos, ir.ONEG, typecheck.Conv(unsafePtr, types.Types[types.TUINTPTR]))) + nif.Cond = ir.NewLogicalExpr(base.Pos, ir.OOROR, overflow, memCond) + nifPtr := ir.NewIfStmt(base.Pos, nil, nil, nil) + nifPtr.Cond = ir.NewBinaryExpr(base.Pos, ir.OEQ, unsafePtr, typecheck.NodNil()) + nifPtr.Body.Append(mkcall("panicunsafeslicenilptr", nil, &nifPtr.Body)) + nif.Body.Append(nifPtr, mkcall("panicunsafeslicelen", nil, &nif.Body)) + appendWalkStmt(init, nif) } - t := n.Type() - - // Call runtime.unsafeslice{,64,checkptr} to check ptr and len. - fn := typecheck.LookupRuntime(fnname) - init.Append(mkcall1(fn, nil, init, reflectdata.TypePtr(t.Elem()), typecheck.Conv(ptr, types.Types[types.TUNSAFEPTR]), typecheck.Conv(len, lenType))) - - h := ir.NewSliceHeaderExpr(n.Pos(), t, + h := ir.NewSliceHeaderExpr(n.Pos(), sliceType, typecheck.Conv(ptr, types.Types[types.TUNSAFEPTR]), typecheck.Conv(len, types.Types[types.TINT]), typecheck.Conv(len, types.Types[types.TINT])) diff --git a/src/runtime/slice.go b/src/runtime/slice.go index e0aeba604f..2413a46d6a 100644 --- a/src/runtime/slice.go +++ b/src/runtime/slice.go @@ -117,6 +117,13 @@ func makeslice64(et *_type, len64, cap64 int64) unsafe.Pointer { return makeslice(et, len, cap) } +// This is a wrapper over runtime/internal/math.MulUintptr, +// so the compiler can recognize and treat it as an intrinsic. +func mulUintptr(a, b uintptr) (uintptr, bool) { + return math.MulUintptr(a, b) +} + +// Keep this code in sync with cmd/compile/internal/walk/builtin.go:walkUnsafeSlice func unsafeslice(et *_type, ptr unsafe.Pointer, len int) { if len < 0 { panicunsafeslicelen() @@ -125,12 +132,13 @@ func unsafeslice(et *_type, ptr unsafe.Pointer, len int) { mem, overflow := math.MulUintptr(et.size, uintptr(len)) if overflow || mem > -uintptr(ptr) { if ptr == nil { - panic(errorString("unsafe.Slice: ptr is nil and len is not zero")) + panicunsafeslicenilptr() } panicunsafeslicelen() } } +// Keep this code in sync with cmd/compile/internal/walk/builtin.go:walkUnsafeSlice func unsafeslice64(et *_type, ptr unsafe.Pointer, len64 int64) { len := int(len64) if int64(len) != len64 { @@ -153,6 +161,10 @@ func panicunsafeslicelen() { panic(errorString("unsafe.Slice: len out of range")) } +func panicunsafeslicenilptr() { + panic(errorString("unsafe.Slice: ptr is nil and len is not zero")) +} + // growslice handles slice growth during append. // It is passed the slice element type, the old slice, and the desired new minimum capacity, // and it returns a new slice with at least that capacity, with the old data