diff --git a/misc/cgo/test/callback.go b/misc/cgo/test/callback.go index 814888e3ac5..08dd9b39d81 100644 --- a/misc/cgo/test/callback.go +++ b/misc/cgo/test/callback.go @@ -182,7 +182,7 @@ func testCallbackCallers(t *testing.T) { "runtime.cgocallbackg1", "runtime.cgocallbackg", "runtime.cgocallback", - "runtime.asmcgocall", + "runtime.systemstack_switch", "runtime.cgocall", "test._Cfunc_callback", "test.nestedCall.func1", diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index 3030101f03d..a59054226c1 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -352,18 +352,12 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-4 // switch stacks // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVL $runtime·systemstack_switch(SB), (g_sched+gobuf_pc)(AX) - MOVL SP, (g_sched+gobuf_sp)(AX) - MOVL AX, (g_sched+gobuf_g)(AX) + CALL gosave_systemstack_switch<>(SB) // switch to g0 get_tls(CX) MOVL DX, g(CX) MOVL (g_sched+gobuf_sp)(DX), BX - // make it look like mstart called systemstack on g0, to stop traceback - SUBL $4, BX - MOVL $runtime·mstart(SB), DX - MOVL DX, 0(BX) MOVL BX, SP // call target function @@ -601,15 +595,18 @@ TEXT runtime·jmpdefer(SB), NOSPLIT, $0-8 MOVL 0(DX), BX JMP BX // but first run the deferred function -// Save state of caller into g->sched. -TEXT gosave<>(SB),NOSPLIT,$0 +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT,$0 PUSHL AX PUSHL BX get_tls(BX) MOVL g(BX), BX LEAL arg+0(FP), AX MOVL AX, (g_sched+gobuf_sp)(BX) - MOVL -4(AX), AX + MOVL $runtime·systemstack_switch(SB), AX MOVL AX, (g_sched+gobuf_pc)(BX) MOVL $0, (g_sched+gobuf_ret)(BX) // Assert ctxt is zero. See func save. @@ -661,7 +658,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-12 JEQ noswitch CMPL DI, m_gsignal(BP) JEQ noswitch - CALL gosave<>(SB) + CALL gosave_systemstack_switch<>(SB) get_tls(CX) MOVL SI, g(CX) MOVL (g_sched+gobuf_sp)(SI), SP diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 9362ce12137..cd5ce3effbd 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -338,20 +338,12 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8 // switch stacks // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVQ $runtime·systemstack_switch(SB), SI - MOVQ SI, (g_sched+gobuf_pc)(AX) - MOVQ SP, (g_sched+gobuf_sp)(AX) - MOVQ AX, (g_sched+gobuf_g)(AX) - MOVQ BP, (g_sched+gobuf_bp)(AX) + CALL gosave_systemstack_switch<>(SB) // switch to g0 MOVQ DX, g(CX) MOVQ DX, R14 // set the g register MOVQ (g_sched+gobuf_sp)(DX), BX - // make it look like mstart called systemstack on g0, to stop traceback - SUBQ $8, BX - MOVQ $runtime·mstart(SB), DX - MOVQ DX, 0(BX) MOVQ BX, SP // call target function @@ -660,13 +652,17 @@ TEXT runtime·jmpdefer(SB), NOSPLIT, $0-16 MOVQ 0(DX), BX JMP BX // but first run the deferred function -// Save state of caller into g->sched. Smashes R9. -TEXT gosave<>(SB),NOSPLIT,$0 +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes R9. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT,$0 #ifndef GOEXPERIMENT_REGABI get_tls(R14) MOVQ g(R14), R14 #endif - MOVQ 0(SP), R9 + MOVQ $runtime·systemstack_switch(SB), R9 MOVQ R9, (g_sched+gobuf_pc)(R14) LEAQ 8(SP), R9 MOVQ R9, (g_sched+gobuf_sp)(R14) @@ -724,7 +720,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20 // Switch to system stack. MOVQ m_g0(R8), SI - CALL gosave<>(SB) + CALL gosave_systemstack_switch<>(SB) MOVQ SI, g(CX) MOVQ (g_sched+gobuf_sp)(SI), SP diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index 109030aada4..c8c53e70dba 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -305,24 +305,14 @@ TEXT runtime·systemstack(SB),NOSPLIT,$0-4 switch: // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVW $runtime·systemstack_switch(SB), R3 - ADD $4, R3, R3 // get past push {lr} - MOVW R3, (g_sched+gobuf_pc)(g) - MOVW R13, (g_sched+gobuf_sp)(g) - MOVW LR, (g_sched+gobuf_lr)(g) - MOVW g, (g_sched+gobuf_g)(g) + BL gosave_systemstack_switch<>(SB) // switch to g0 MOVW R0, R5 MOVW R2, R0 BL setg<>(SB) MOVW R5, R0 - MOVW (g_sched+gobuf_sp)(R2), R3 - // make it look like mstart called systemstack on g0, to stop traceback - SUB $4, R3, R3 - MOVW $runtime·mstart(SB), R4 - MOVW R4, 0(R3) - MOVW R3, R13 + MOVW (g_sched+gobuf_sp)(R2), R13 // call target function MOVW R0, R7 @@ -537,19 +527,25 @@ TEXT runtime·jmpdefer(SB),NOSPLIT,$0-8 MOVW 0(R7), R1 B (R1) -// Save state of caller into g->sched. Smashes R11. -TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 - MOVW LR, (g_sched+gobuf_pc)(g) +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes R11. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 + MOVW $runtime·systemstack_switch(SB), R11 + ADD $4, R11 // get past push {lr} + MOVW R11, (g_sched+gobuf_pc)(g) MOVW R13, (g_sched+gobuf_sp)(g) + MOVW g, (g_sched+gobuf_g)(g) MOVW $0, R11 MOVW R11, (g_sched+gobuf_lr)(g) MOVW R11, (g_sched+gobuf_ret)(g) - MOVW R11, (g_sched+gobuf_ctxt)(g) // Assert ctxt is zero. See func save. MOVW (g_sched+gobuf_ctxt)(g), R11 - CMP $0, R11 + TST R11, R11 B.EQ 2(PC) - CALL runtime·badctxt(SB) + BL runtime·badctxt(SB) RET // func asmcgocall_no_g(fn, arg unsafe.Pointer) @@ -590,7 +586,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-12 MOVW m_g0(R8), R3 CMP R3, g BEQ nosave - BL gosave<>(SB) + BL gosave_systemstack_switch<>(SB) MOVW R0, R5 MOVW R3, R0 BL setg<>(SB) diff --git a/src/runtime/asm_arm64.s b/src/runtime/asm_arm64.s index 79efd4cb17a..31a6fe57b9b 100644 --- a/src/runtime/asm_arm64.s +++ b/src/runtime/asm_arm64.s @@ -205,24 +205,12 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8 switch: // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVD $runtime·systemstack_switch(SB), R6 - ADD $8, R6 // get past prologue - MOVD R6, (g_sched+gobuf_pc)(g) - MOVD RSP, R0 - MOVD R0, (g_sched+gobuf_sp)(g) - MOVD R29, (g_sched+gobuf_bp)(g) - MOVD $0, (g_sched+gobuf_lr)(g) - MOVD g, (g_sched+gobuf_g)(g) + BL gosave_systemstack_switch<>(SB) // switch to g0 MOVD R5, g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R3 - // make it look like mstart called systemstack on g0, to stop traceback - SUB $16, R3 - AND $~15, R3 - MOVD $runtime·mstart(SB), R4 - MOVD R4, 0(R3) MOVD R3, RSP MOVD (g_sched+gobuf_bp)(g), R29 @@ -859,14 +847,21 @@ TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16 MOVD 0(R26), R3 B (R3) -// Save state of caller into g->sched. Smashes R0. -TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 - MOVD LR, (g_sched+gobuf_pc)(g) +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes R0. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 + MOVD $runtime·systemstack_switch(SB), R0 + ADD $8, R0 // get past prologue + MOVD R0, (g_sched+gobuf_pc)(g) MOVD RSP, R0 MOVD R0, (g_sched+gobuf_sp)(g) MOVD R29, (g_sched+gobuf_bp)(g) MOVD $0, (g_sched+gobuf_lr)(g) MOVD $0, (g_sched+gobuf_ret)(g) + MOVD g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOVD (g_sched+gobuf_ctxt)(g), R0 CBZ R0, 2(PC) @@ -908,8 +903,8 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20 BEQ nosave // Switch to system stack. - MOVD R0, R9 // gosave<> and save_g might clobber R0 - BL gosave<>(SB) + MOVD R0, R9 // gosave_systemstack_switch<> and save_g might clobber R0 + BL gosave_systemstack_switch<>(SB) MOVD R3, g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R0 diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s index 6e1d25cd90c..75bb2230664 100644 --- a/src/runtime/asm_mips64x.s +++ b/src/runtime/asm_mips64x.s @@ -169,21 +169,12 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8 switch: // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVV $runtime·systemstack_switch(SB), R4 - ADDV $8, R4 // get past prologue - MOVV R4, (g_sched+gobuf_pc)(g) - MOVV R29, (g_sched+gobuf_sp)(g) - MOVV R0, (g_sched+gobuf_lr)(g) - MOVV g, (g_sched+gobuf_g)(g) + JAL gosave_systemstack_switch<>(SB) // switch to g0 MOVV R3, g JAL runtime·save_g(SB) MOVV (g_sched+gobuf_sp)(g), R1 - // make it look like mstart called systemstack on g0, to stop traceback - ADDV $-8, R1 - MOVV $runtime·mstart(SB), R2 - MOVV R2, 0(R1) MOVV R1, R29 // call target function @@ -401,12 +392,19 @@ TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16 MOVV 0(REGCTXT), R4 JMP (R4) -// Save state of caller into g->sched. Smashes R1. -TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 - MOVV R31, (g_sched+gobuf_pc)(g) +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes R1. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 + MOVV $runtime·systemstack_switch(SB), R1 + ADDV $8, R1 // get past prologue + MOVV R1, (g_sched+gobuf_pc)(g) MOVV R29, (g_sched+gobuf_sp)(g) MOVV R0, (g_sched+gobuf_lr)(g) MOVV R0, (g_sched+gobuf_ret)(g) + MOVV g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOVV (g_sched+gobuf_ctxt)(g), R1 BEQ R1, 2(PC) @@ -440,7 +438,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20 MOVV m_g0(R5), R6 BEQ R6, g, g0 - JAL gosave<>(SB) + JAL gosave_systemstack_switch<>(SB) MOVV R6, g JAL runtime·save_g(SB) MOVV (g_sched+gobuf_sp)(g), R29 diff --git a/src/runtime/asm_mipsx.s b/src/runtime/asm_mipsx.s index 8e5753d2557..341da8e8d71 100644 --- a/src/runtime/asm_mipsx.s +++ b/src/runtime/asm_mipsx.s @@ -170,21 +170,12 @@ TEXT runtime·systemstack(SB),NOSPLIT,$0-4 switch: // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVW $runtime·systemstack_switch(SB), R4 - ADDU $8, R4 // get past prologue - MOVW R4, (g_sched+gobuf_pc)(g) - MOVW R29, (g_sched+gobuf_sp)(g) - MOVW R0, (g_sched+gobuf_lr)(g) - MOVW g, (g_sched+gobuf_g)(g) + JAL gosave_systemstack_switch<>(SB) // switch to g0 MOVW R3, g JAL runtime·save_g(SB) MOVW (g_sched+gobuf_sp)(g), R1 - // make it look like mstart called systemstack on g0, to stop traceback - ADDU $-4, R1 - MOVW $runtime·mstart(SB), R2 - MOVW R2, 0(R1) MOVW R1, R29 // call target function @@ -401,12 +392,19 @@ TEXT runtime·jmpdefer(SB),NOSPLIT,$0-8 MOVW 0(REGCTXT), R4 JMP (R4) -// Save state of caller into g->sched. Smashes R1. -TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 - MOVW R31, (g_sched+gobuf_pc)(g) +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes R1. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 + MOVW $runtime·systemstack_switch(SB), R1 + ADDU $8, R1 // get past prologue + MOVW R1, (g_sched+gobuf_pc)(g) MOVW R29, (g_sched+gobuf_sp)(g) MOVW R0, (g_sched+gobuf_lr)(g) MOVW R0, (g_sched+gobuf_ret)(g) + MOVW g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOVW (g_sched+gobuf_ctxt)(g), R1 BEQ R1, 2(PC) @@ -431,7 +429,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-12 MOVW m_g0(R5), R6 BEQ R6, g, g0 - JAL gosave<>(SB) + JAL gosave_systemstack_switch<>(SB) MOVW R6, g JAL runtime·save_g(SB) MOVW (g_sched+gobuf_sp)(g), R29 diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s index 834023cce19..a99a61fd88a 100644 --- a/src/runtime/asm_ppc64x.s +++ b/src/runtime/asm_ppc64x.s @@ -229,22 +229,12 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8 switch: // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVD $runtime·systemstack_switch(SB), R6 - ADD $16, R6 // get past prologue (including r2-setting instructions when they're there) - MOVD R6, (g_sched+gobuf_pc)(g) - MOVD R1, (g_sched+gobuf_sp)(g) - MOVD R0, (g_sched+gobuf_lr)(g) - MOVD g, (g_sched+gobuf_g)(g) + BL gosave_systemstack_switch<>(SB) // switch to g0 MOVD R5, g BL runtime·save_g(SB) - MOVD (g_sched+gobuf_sp)(g), R3 - // make it look like mstart called systemstack on g0, to stop traceback - SUB $FIXED_FRAME, R3 - MOVD $runtime·mstart(SB), R4 - MOVD R4, 0(R3) - MOVD R3, R1 + MOVD (g_sched+gobuf_sp)(g), R1 // call target function MOVD 0(R11), R12 // code pointer @@ -534,13 +524,19 @@ TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16 MOVD R12, CTR BR (CTR) -// Save state of caller into g->sched. Smashes R31. -TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 - MOVD LR, R31 +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes R31. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 + MOVD $runtime·systemstack_switch(SB), R31 + ADD $16, R31 // get past prologue (including r2-setting instructions when they're there) MOVD R31, (g_sched+gobuf_pc)(g) MOVD R1, (g_sched+gobuf_sp)(g) MOVD R0, (g_sched+gobuf_lr)(g) MOVD R0, (g_sched+gobuf_ret)(g) + MOVD g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOVD (g_sched+gobuf_ctxt)(g), R31 CMP R0, R31 @@ -577,7 +573,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20 MOVD m_g0(R8), R6 CMP R6, g BEQ g0 - BL gosave<>(SB) + BL gosave_systemstack_switch<>(SB) MOVD R6, g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R1 diff --git a/src/runtime/asm_riscv64.s b/src/runtime/asm_riscv64.s index 3d0349471a2..fb7c6530dcd 100644 --- a/src/runtime/asm_riscv64.s +++ b/src/runtime/asm_riscv64.s @@ -114,21 +114,12 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8 switch: // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOV $runtime·systemstack_switch(SB), T2 - ADD $8, T2 // get past prologue - MOV T2, (g_sched+gobuf_pc)(g) - MOV X2, (g_sched+gobuf_sp)(g) - MOV ZERO, (g_sched+gobuf_lr)(g) - MOV g, (g_sched+gobuf_g)(g) + CALL gosave_systemstack_switch<>(SB) // switch to g0 MOV T1, g CALL runtime·save_g(SB) MOV (g_sched+gobuf_sp)(g), T0 - // make it look like mstart called systemstack on g0, to stop traceback - ADD $-8, T0 - MOV $runtime·mstart(SB), T1 - MOV T1, 0(T0) MOV T0, X2 // call target function @@ -297,12 +288,19 @@ TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8 JALR RA, T1 JMP runtime·badmcall2(SB) -// Save state of caller into g->sched. Smashes X31. -TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 - MOV X1, (g_sched+gobuf_pc)(g) +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes X31. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 + MOV $runtime·systemstack_switch(SB), X31 + ADD $8, X31 // get past prologue + MOV X31, (g_sched+gobuf_pc)(g) MOV X2, (g_sched+gobuf_sp)(g) MOV ZERO, (g_sched+gobuf_lr)(g) MOV ZERO, (g_sched+gobuf_ret)(g) + MOV g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOV (g_sched+gobuf_ctxt)(g), X31 BEQ ZERO, X31, 2(PC) @@ -327,7 +325,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20 MOV m_g0(X6), X7 BEQ X7, g, g0 - CALL gosave<>(SB) + CALL gosave_systemstack_switch<>(SB) MOV X7, g CALL runtime·save_g(SB) MOV (g_sched+gobuf_sp)(g), X2 diff --git a/src/runtime/asm_s390x.s b/src/runtime/asm_s390x.s index fbd185c3533..43244d961fb 100644 --- a/src/runtime/asm_s390x.s +++ b/src/runtime/asm_s390x.s @@ -256,22 +256,12 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8 switch: // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVD $runtime·systemstack_switch(SB), R6 - ADD $16, R6 // get past prologue - MOVD R6, (g_sched+gobuf_pc)(g) - MOVD R15, (g_sched+gobuf_sp)(g) - MOVD $0, (g_sched+gobuf_lr)(g) - MOVD g, (g_sched+gobuf_g)(g) + BL gosave_systemstack_switch<>(SB) // switch to g0 MOVD R5, g BL runtime·save_g(SB) - MOVD (g_sched+gobuf_sp)(g), R3 - // make it look like mstart called systemstack on g0, to stop traceback - SUB $8, R3 - MOVD $runtime·mstart(SB), R4 - MOVD R4, 0(R3) - MOVD R3, R15 + MOVD (g_sched+gobuf_sp)(g), R15 // call target function MOVD 0(R12), R3 // code pointer @@ -498,12 +488,19 @@ TEXT runtime·jmpdefer(SB),NOSPLIT|NOFRAME,$0-16 MOVD 0(R12), R3 BR (R3) -// Save state of caller into g->sched. Smashes R1. -TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 - MOVD LR, (g_sched+gobuf_pc)(g) +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes R1. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 + MOVD $runtime·systemstack_switch(SB), R1 + ADD $16, R1 // get past prologue + MOVD R1, (g_sched+gobuf_pc)(g) MOVD R15, (g_sched+gobuf_sp)(g) MOVD $0, (g_sched+gobuf_lr)(g) MOVD $0, (g_sched+gobuf_ret)(g) + MOVD g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOVD (g_sched+gobuf_ctxt)(g), R1 CMPBEQ R1, $0, 2(PC) @@ -529,7 +526,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20 MOVD g_m(g), R6 MOVD m_g0(R6), R6 CMPBEQ R6, g, g0 - BL gosave<>(SB) + BL gosave_systemstack_switch<>(SB) MOVD R6, g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R15 diff --git a/src/runtime/sys_windows_386.s b/src/runtime/sys_windows_386.s index c556e3a3c25..576dd287719 100644 --- a/src/runtime/sys_windows_386.s +++ b/src/runtime/sys_windows_386.s @@ -94,7 +94,7 @@ TEXT sigtramp<>(SB),NOSPLIT,$0-0 JNE 2(PC) CALL runtime·badsignal2(SB) - // save g and SP in case of stack switch + // save g in case of stack switch MOVL DX, 32(SP) // g MOVL SP, 36(SP) @@ -108,13 +108,9 @@ TEXT sigtramp<>(SB),NOSPLIT,$0-0 get_tls(BP) MOVL BX, g(BP) MOVL (g_sched+gobuf_sp)(BX), DI - // make it look like mstart called us on g0, to stop traceback - SUBL $4, DI - MOVL $runtime·mstart(SB), 0(DI) - // traceback will think that we've done SUBL - // on this stack, so subtract them here to match. - // (we need room for sighandler arguments anyway). + // make room for sighandler arguments // and re-save old SP for restoring later. + // (note that the 36(DI) here must match the 36(SP) above.) SUBL $40, DI MOVL SP, 36(DI) MOVL DI, SP @@ -132,7 +128,7 @@ g0: // switch back to original stack and g // no-op if we never left. MOVL 36(SP), SP - MOVL 32(SP), DX + MOVL 32(SP), DX // note: different SP get_tls(BP) MOVL DX, g(BP) diff --git a/src/runtime/sys_windows_amd64.s b/src/runtime/sys_windows_amd64.s index 9cd14016b01..6f2abb5444b 100644 --- a/src/runtime/sys_windows_amd64.s +++ b/src/runtime/sys_windows_amd64.s @@ -151,16 +151,10 @@ TEXT sigtramp<>(SB),NOSPLIT|NOFRAME,$0-0 get_tls(BP) MOVQ BX, g(BP) MOVQ (g_sched+gobuf_sp)(BX), DI - // make it look like mstart called us on g0, to stop traceback - SUBQ $8, DI - MOVQ $runtime·mstart(SB), SI - MOVQ SI, 0(DI) - // traceback will think that we've done PUSHFQ and SUBQ - // on this stack, so subtract them here to match. - // (we need room for sighandler arguments anyway). + // make room for sighandler arguments // and re-save old SP for restoring later. - SUBQ $(112+8), DI - // save g, save old stack pointer. + // (note that the 104(DI) here must match the 104(SP) above.) + SUBQ $120, DI MOVQ SP, 104(DI) MOVQ DI, SP diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s index d2bdc50e3b6..7d134dd3f41 100644 --- a/src/runtime/sys_windows_arm.s +++ b/src/runtime/sys_windows_arm.s @@ -6,6 +6,8 @@ #include "go_tls.h" #include "textflag.h" +// Note: For system ABI, R0-R3 are args, R4-R11 are callee-save. + // void runtime·asmstdcall(void *c); TEXT runtime·asmstdcall(SB),NOSPLIT|NOFRAME,$0 MOVM.DB.W [R4, R5, R14], (R13) // push {r4, r5, lr} @@ -139,11 +141,10 @@ TEXT sigtramp<>(SB),NOSPLIT|NOFRAME,$0 MOVW (g_sched+gobuf_sp)(g), R3 // R3 = g->gobuf.sp BL runtime·save_g(SB) - // traceback will think that we've done PUSH and SUB - // on this stack, so subtract them here to match. - // (we need room for sighandler arguments anyway). + // make room for sighandler arguments // and re-save old SP for restoring later. - SUB $(40+8+20), R3 + // (note that the 24(R3) here must match the 24(R13) above.) + SUB $40, R3 MOVW R13, 24(R3) // save old stack pointer MOVW R3, R13 // switch stack @@ -151,22 +152,14 @@ g0: MOVW 0(R6), R2 // R2 = ExceptionPointers->ExceptionRecord MOVW 4(R6), R3 // R3 = ExceptionPointers->ContextRecord - // make it look like mstart called us on g0, to stop traceback - MOVW $runtime·mstart(SB), R4 - - MOVW R4, 0(R13) // Save link register for traceback + MOVW $0, R4 + MOVW R4, 0(R13) // No saved link register. MOVW R2, 4(R13) // Move arg0 (ExceptionRecord) into position MOVW R3, 8(R13) // Move arg1 (ContextRecord) into position MOVW R5, 12(R13) // Move arg2 (original g) into position BL (R7) // Call the go routine MOVW 16(R13), R4 // Fetch return value from stack - // Compute the value of the g0 stack pointer after deallocating - // this frame, then allocating 8 bytes. We may need to store - // the resume SP and PC on the g0 stack to work around - // control flow guard when we resume from the exception. - ADD $(40+20), R13, R12 - // switch back to original stack and g MOVW 24(R13), R13 MOVW 20(R13), g @@ -192,33 +185,36 @@ done: // If returntramp has already been set up by a previous exception // handler, don't clobber the stored SP and PC on the stack. MOVW 4(R3), R3 // PEXCEPTION_POINTERS->Context - MOVW 0x40(R3), R2 // load PC from context record + MOVW context_pc(R3), R2 // load PC from context record MOVW $returntramp<>(SB), R1 CMP R1, R2 B.EQ return // do not clobber saved SP/PC - // Save resume SP and PC on g0 stack - MOVW 0x38(R3), R2 // load SP from context record - MOVW R2, 0(R12) // Store resume SP on g0 stack - MOVW 0x40(R3), R2 // load PC from context record - MOVW R2, 4(R12) // Store resume PC on g0 stack + // Save resume SP and PC into R0, R1. + MOVW context_spr(R3), R2 + MOVW R2, context_r0(R3) + MOVW context_pc(R3), R2 + MOVW R2, context_r1(R3) // Set up context record to return to returntramp on g0 stack - MOVW R12, 0x38(R3) // save g0 stack pointer - // in context record - MOVW $returntramp<>(SB), R2 // save resume address - MOVW R2, 0x40(R3) // in context record + MOVW R12, context_spr(R3) + MOVW $returntramp<>(SB), R2 + MOVW R2, context_pc(R3) return: B (R14) // return -// // Trampoline to resume execution from exception handler. // This is part of the control flow guard workaround. // It switches stacks and jumps to the continuation address. -// +// R0 and R1 are set above at the end of sigtramp<> +// in the context that starts executing at returntramp<>. TEXT returntramp<>(SB),NOSPLIT|NOFRAME,$0 - MOVM.IA (R13), [R13, R15] // ldm sp, [sp, pc] + // Important: do not smash LR, + // which is set to a live value when handling + // a signal by pushing a call to sigpanic onto the stack. + MOVW R0, R13 + B (R1) TEXT runtime·exceptiontramp(SB),NOSPLIT|NOFRAME,$0 MOVW $runtime·exceptionhandler(SB), R1