qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-09-23 11:20:17 -06:00
Code Releases Activity

runtime/cgo: store M for C-created thread in pthread key

Browse Source 
In a C thread, it's necessary to acquire an extra M by using needm while invoking a Go function from C. But, needm and dropm are heavy costs due to the signal-related syscalls.
So, we change to not dropm while returning back to C, which means binding the extra M to the C thread until it exits, to avoid needm and dropm on each C to Go call.
Instead, we only dropm while the C thread exits, so the extra M won't leak.

When invoking a Go function from C:
Allocate a pthread variable using pthread_key_create, only once per shared object, and register a thread-exit-time destructor.
And store the g0 of the current m into the thread-specified value of the pthread key,  only once per C thread, so that the destructor will put the extra M back onto the extra M list while the C thread exits.

When returning back to C:
Skip dropm in cgocallback, when the pthread variable has been created, so that the extra M will be reused the next time invoke a Go function from C.

This is purely a performance optimization. The old version, in which needm & dropm happen on each cgo call, is still correct too, and we have to keep the old version on systems with cgo but without pthreads, like Windows.

This optimization is significant, and the specific value depends on the OS system and CPU, but in general, it can be considered as 10x faster, for a simple Go function call from a C thread.

For the newly added BenchmarkCGoInCThread, some benchmark results:
1. it's 28x faster, from 3395 ns/op to 121 ns/op, in darwin OS & Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz
2. it's 6.5x faster, from 1495 ns/op to 230 ns/op, in Linux OS & Intel(R) Xeon(R) CPU E5-2630 0 @ 2.30GHz

Fixes #51676

Change-Id: I380702fe2f9b6b401b2d6f04b0aba990f4b9ee6c
GitHub-Last-Rev: 93dc64ad98
GitHub-Pull-Request: golang/go#51679
Reviewed-on: https://go-review.googlesource.com/c/go/+/392854
Reviewed-by: Ian Lance Taylor <iant@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: thepudds <thepudds1460@gmail.com>
Reviewed-by: Cherry Mui <cherryyz@google.com>
This commit is contained in:
doujiang24 2023-03-24 01:30:46 +00:00 committed by Cherry Mui
parent a6c382eaa8
commit ef0dedce87
37 changed files with 760 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
misc/cgo/test/cgo_test.go
View File

@ -104,6 +104,7 @@ func TestThreadLock(t *testing.T) { testThreadLockFunc(t) }
func TestUnsignedInt(t *testing.T) { testUnsignedInt(t) }
func TestZeroArgCallback(t *testing.T) { testZeroArgCallback(t) }
func BenchmarkCgoCall(b *testing.B) { benchCgoCall(b) }
func BenchmarkGoString(b *testing.B) { benchGoString(b) }
func BenchmarkCGoCallback(b *testing.B) { benchCallback(b) }
func BenchmarkCgoCall(b *testing.B) { benchCgoCall(b) }
func BenchmarkGoString(b *testing.B) { benchGoString(b) }
func BenchmarkCGoCallback(b *testing.B) { benchCallback(b) }
func BenchmarkCGoInCThread(b *testing.B) { benchCGoInCthread(b) }

24
misc/cgo/test/cthread_unix.c
View File

@ -32,3 +32,27 @@ doAdd(int max, int nthread)
for(i=0; i<nthread; i++)
pthread_join(thread_id[i], 0);
}
static void*
goDummyCallbackThread(void* p)
{
int i, max;
max = *(int*)p;
for(i=0; i<max; i++)
goDummy();
return NULL;
}
int
callGoInCThread(int max)
{
pthread_t thread;
if (pthread_create(&thread, NULL, goDummyCallbackThread, (void*)(&max)) != 0)
return -1;
if (pthread_join(thread, NULL) != 0)
return -1;
return max;
}

22
misc/cgo/test/cthread_windows.c
View File

@ -35,3 +35,25 @@ doAdd(int max, int nthread)
CloseHandle((HANDLE)thread_id[i]);
}
}
__stdcall
static unsigned int
goDummyCallbackThread(void* p)
{
int i, max;
max = *(int*)p;
for(i=0; i<max; i++)
goDummy();
return 0;
}
int
callGoInCThread(int max)
{
uintptr_t thread_id;
thread_id = _beginthreadex(0, 0, goDummyCallbackThread, &max, 0, 0);
WaitForSingleObject((HANDLE)thread_id, INFINITE);
CloseHandle((HANDLE)thread_id);
return max;
}

14
misc/cgo/test/testx.go
View File

@ -24,6 +24,7 @@ import (
/*
// threads
extern void doAdd(int, int);
extern int callGoInCThread(int);
// issue 1328
void IntoC(void);
@ -146,6 +147,10 @@ func Add(x int) {
*p = 2
}
//export goDummy
func goDummy() {
}
func testCthread(t *testing.T) {
if (runtime.GOOS == "darwin" || runtime.GOOS == "ios") && runtime.GOARCH == "arm64" {
t.Skip("the iOS exec wrapper is unable to properly handle the panic from Add")
@ -159,6 +164,15 @@ func testCthread(t *testing.T) {
}
}
// Benchmark measuring overhead from C to Go in a C thread.
// Create a new C thread and invoke Go function repeatedly in the new C thread.
func benchCGoInCthread(b *testing.B) {
n := C.callGoInCThread(C.int(b.N))
if int(n) != b.N {
b.Fatal("unmatch loop times")
}
}
// issue 1328
//export BackIntoGo

41
src/runtime/asm_386.s
View File

@ -689,7 +689,20 @@ nosave:
TEXT ·cgocallback(SB),NOSPLIT,$12-12 // Frame size must match commented places below
NO_LOCAL_POINTERS
// If g is nil, Go did not create the current thread.
// Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
// It is used to dropm while thread is exiting.
MOVL fn+0(FP), AX
CMPL AX, $0
JNE loadg
// Restore the g from frame.
get_tls(CX)
MOVL frame+4(FP), BX
MOVL BX, g(CX)
JMP dropm
loadg:
// If g is nil, Go did not create the current thread,
// or if this thread never called into Go on pthread platforms.
// Call needm to obtain one for temporary use.
// In this case, we're running on the thread stack, so there's
// lots of space, but the linker doesn't know. Hide the call from
@ -707,9 +720,9 @@ TEXT ·cgocallback(SB),NOSPLIT,$12-12 // Frame size must match commented places
MOVL BP, savedm-4(SP) // saved copy of oldm
JMP havem
needm:
MOVL $runtime·needm(SB), AX
MOVL $runtime·needAndBindM(SB), AX
CALL AX
MOVL $0, savedm-4(SP) // dropm on return
MOVL $0, savedm-4(SP)
get_tls(CX)
MOVL g(CX), BP
MOVL g_m(BP), BP
@ -784,13 +797,29 @@ havem:
MOVL 0(SP), AX
MOVL AX, (g_sched+gobuf_sp)(SI)
// If the m on entry was nil, we called needm above to borrow an m
// for the duration of the call. Since the call is over, return it with dropm.
// If the m on entry was nil, we called needm above to borrow an m,
// 1. for the duration of the call on non-pthread platforms,
// 2. or the duration of the C thread alive on pthread platforms.
// If the m on entry wasn't nil,
// 1. the thread might be a Go thread,
// 2. or it's wasn't the first call from a C thread on pthread platforms,
// since the we skip dropm to resue the m in the first call.
MOVL savedm-4(SP), DX
CMPL DX, $0
JNE 3(PC)
JNE droppedm
// Skip dropm to reuse it in the next call, when a pthread key has been created.
MOVL _cgo_pthread_key_created(SB), DX
// It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
CMPL DX, $0
JEQ dropm
CMPL (DX), $0
JNE droppedm
dropm:
MOVL $runtime·dropm(SB), AX
CALL AX
droppedm:
// Done!
RET

38
src/runtime/asm_amd64.s
View File

@ -915,7 +915,20 @@ GLOBL zeroTLS<>(SB),RODATA,$const_tlsSize
TEXT ·cgocallback(SB),NOSPLIT,$24-24
NO_LOCAL_POINTERS
// If g is nil, Go did not create the current thread.
// Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
// It is used to dropm while thread is exiting.
MOVQ fn+0(FP), AX
CMPQ AX, $0
JNE loadg
// Restore the g from frame.
get_tls(CX)
MOVQ frame+8(FP), BX
MOVQ BX, g(CX)
JMP dropm
loadg:
// If g is nil, Go did not create the current thread,
// or if this thread never called into Go on pthread platforms.
// Call needm to obtain one m for temporary use.
// In this case, we're running on the thread stack, so there's
// lots of space, but the linker doesn't know. Hide the call from
@ -953,9 +966,9 @@ needm:
// a bad value in there, in case needm tries to use it.
XORPS X15, X15
XORQ R14, R14
MOVQ $runtime·needm<ABIInternal>(SB), AX
MOVQ $runtime·needAndBindM<ABIInternal>(SB), AX
CALL AX
MOVQ $0, savedm-8(SP) // dropm on return
MOVQ $0, savedm-8(SP)
get_tls(CX)
MOVQ g(CX), BX
MOVQ g_m(BX), BX
@ -1044,11 +1057,26 @@ havem:
MOVQ 0(SP), AX
MOVQ AX, (g_sched+gobuf_sp)(SI)
// If the m on entry was nil, we called needm above to borrow an m
// for the duration of the call. Since the call is over, return it with dropm.
// If the m on entry was nil, we called needm above to borrow an m,
// 1. for the duration of the call on non-pthread platforms,
// 2. or the duration of the C thread alive on pthread platforms.
// If the m on entry wasn't nil,
// 1. the thread might be a Go thread,
// 2. or it's wasn't the first call from a C thread on pthread platforms,
// since the we skip dropm to resue the m in the first call.
MOVQ savedm-8(SP), BX
CMPQ BX, $0
JNE done
// Skip dropm to reuse it in the next call, when a pthread key has been created.
MOVQ _cgo_pthread_key_created(SB), AX
// It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
CMPQ AX, $0
JEQ dropm
CMPQ (AX), $0
JNE done
dropm:
MOVQ $runtime·dropm(SB), AX
CALL AX
#ifdef GOOS_windows

37
src/runtime/asm_arm.s
View File

@ -630,6 +630,15 @@ nosave:
TEXT ·cgocallback(SB),NOSPLIT,$12-12
NO_LOCAL_POINTERS
// Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
// It is used to dropm while thread is exiting.
MOVW fn+0(FP), R1
B.NE loadg
// Restore the g from frame.
MOVW frame+4(FP), g
B dropm
loadg:
// Load m and g from thread-local storage.
#ifdef GOOS_openbsd
BL runtime·load_g(SB)
@ -639,7 +648,8 @@ TEXT ·cgocallback(SB),NOSPLIT,$12-12
BL.NE runtime·load_g(SB)
#endif
// If g is nil, Go did not create the current thread.
// If g is nil, Go did not create the current thread,
// or if this thread never called into Go on pthread platforms.
// Call needm to obtain one for temporary use.
// In this case, we're running on the thread stack, so there's
// lots of space, but the linker doesn't know. Hide the call from
@ -653,7 +663,7 @@ TEXT ·cgocallback(SB),NOSPLIT,$12-12
needm:
MOVW g, savedm-4(SP) // g is zero, so is m.
MOVW $runtime·needm(SB), R0
MOVW $runtime·needAndBindM(SB), R0
BL (R0)
// Set m->g0->sched.sp = SP, so that if a panic happens
@ -724,14 +734,31 @@ havem:
MOVW savedsp-12(SP), R4 // must match frame size
MOVW R4, (g_sched+gobuf_sp)(g)
// If the m on entry was nil, we called needm above to borrow an m
// for the duration of the call. Since the call is over, return it with dropm.
// If the m on entry was nil, we called needm above to borrow an m,
// 1. for the duration of the call on non-pthread platforms,
// 2. or the duration of the C thread alive on pthread platforms.
// If the m on entry wasn't nil,
// 1. the thread might be a Go thread,
// 2. or it's wasn't the first call from a C thread on pthread platforms,
// since the we skip dropm to resue the m in the first call.
MOVW savedm-4(SP), R6
CMP $0, R6
B.NE 3(PC)
B.NE done
// Skip dropm to reuse it in the next call, when a pthread key has been created.
MOVW _cgo_pthread_key_created(SB), R6
// It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
CMP $0, R6
B.EQ dropm
MOVW (R6), R6
CMP $0, R6
B.NE done
dropm:
MOVW $runtime·dropm(SB), R0
BL (R0)
done:
// Done!
RET

32
src/runtime/asm_arm64.s
View File

@ -1014,10 +1014,20 @@ nosave:
TEXT ·cgocallback(SB),NOSPLIT,$24-24
NO_LOCAL_POINTERS
// Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
// It is used to dropm while thread is exiting.
MOVD fn+0(FP), R1
CBNZ R1, loadg
// Restore the g from frame.
MOVD frame+8(FP), g
B dropm
loadg:
// Load g from thread-local storage.
BL runtime·load_g(SB)
// If g is nil, Go did not create the current thread.
// If g is nil, Go did not create the current thread,
// or if this thread never called into Go on pthread platforms.
// Call needm to obtain one for temporary use.
// In this case, we're running on the thread stack, so there's
// lots of space, but the linker doesn't know. Hide the call from
@ -1030,7 +1040,7 @@ TEXT ·cgocallback(SB),NOSPLIT,$24-24
needm:
MOVD g, savedm-8(SP) // g is zero, so is m.
MOVD $runtime·needm(SB), R0
MOVD $runtime·needAndBindM(SB), R0
BL (R0)
// Set m->g0->sched.sp = SP, so that if a panic happens
@ -1111,10 +1121,24 @@ havem:
MOVD savedsp-16(SP), R4
MOVD R4, (g_sched+gobuf_sp)(g)
// If the m on entry was nil, we called needm above to borrow an m
// for the duration of the call. Since the call is over, return it with dropm.
// If the m on entry was nil, we called needm above to borrow an m,
// 1. for the duration of the call on non-pthread platforms,
// 2. or the duration of the C thread alive on pthread platforms.
// If the m on entry wasn't nil,
// 1. the thread might be a Go thread,
// 2. or it's wasn't the first call from a C thread on pthread platforms,
// since the we skip dropm to resue the m in the first call.
MOVD savedm-8(SP), R6
CBNZ R6, droppedm
// Skip dropm to reuse it in the next call, when a pthread key has been created.
MOVD _cgo_pthread_key_created(SB), R6
// It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
CBZ R6, dropm
MOVD (R6), R6
CBNZ R6, droppedm
dropm:
MOVD $runtime·dropm(SB), R0
BL (R0)
droppedm:

32
src/runtime/asm_loong64.s
View File

@ -461,13 +461,23 @@ g0:
TEXT ·cgocallback(SB),NOSPLIT,$24-24
NO_LOCAL_POINTERS
// Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
// It is used to dropm while thread is exiting.
MOVV fn+0(FP), R5
BNE R5, loadg
// Restore the g from frame.
MOVV frame+8(FP), g
JMP dropm
loadg:
// Load m and g from thread-local storage.
MOVB runtime·iscgo(SB), R19
BEQ R19, nocgo
JAL runtime·load_g(SB)
nocgo:
// If g is nil, Go did not create the current thread.
// If g is nil, Go did not create the current thread,
// or if this thread never called into Go on pthread platforms.
// Call needm to obtain one for temporary use.
// In this case, we're running on the thread stack, so there's
// lots of space, but the linker doesn't know. Hide the call from
@ -480,7 +490,7 @@ nocgo:
needm:
MOVV g, savedm-8(SP) // g is zero, so is m.
MOVV $runtime·needm(SB), R4
MOVV $runtime·needAndBindM(SB), R4
JAL (R4)
// Set m->sched.sp = SP, so that if a panic happens
@ -552,10 +562,24 @@ havem:
MOVV savedsp-24(SP), R13 // must match frame size
MOVV R13, (g_sched+gobuf_sp)(g)
// If the m on entry was nil, we called needm above to borrow an m
// for the duration of the call. Since the call is over, return it with dropm.
// If the m on entry was nil, we called needm above to borrow an m,
// 1. for the duration of the call on non-pthread platforms,
// 2. or the duration of the C thread alive on pthread platforms.
// If the m on entry wasn't nil,
// 1. the thread might be a Go thread,
// 2. or it's wasn't the first call from a C thread on pthread platforms,
// since the we skip dropm to resue the m in the first call.
MOVV savedm-8(SP), R12
BNE R12, droppedm
// Skip dropm to reuse it in the next call, when a pthread key has been created.
MOVV _cgo_pthread_key_created(SB), R12
// It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
BEQ R12, dropm
MOVV (R12), R12
BNE R12, droppedm
dropm:
MOVV $runtime·dropm(SB), R4
JAL (R4)
droppedm:

32
src/runtime/asm_mips64x.s
View File

@ -469,13 +469,23 @@ g0:
TEXT ·cgocallback(SB),NOSPLIT,$24-24
NO_LOCAL_POINTERS
// Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
// It is used to dropm while thread is exiting.
MOVV fn+0(FP), R5
BNE R5, loadg
// Restore the g from frame.
MOVV frame+8(FP), g
JMP dropm
loadg:
// Load m and g from thread-local storage.
MOVB runtime·iscgo(SB), R1
BEQ R1, nocgo
JAL runtime·load_g(SB)
nocgo:
// If g is nil, Go did not create the current thread.
// If g is nil, Go did not create the current thread,
// or if this thread never called into Go on pthread platforms.
// Call needm to obtain one for temporary use.
// In this case, we're running on the thread stack, so there's
// lots of space, but the linker doesn't know. Hide the call from
@ -488,7 +498,7 @@ nocgo:
needm:
MOVV g, savedm-8(SP) // g is zero, so is m.
MOVV $runtime·needm(SB), R4
MOVV $runtime·needAndBindM(SB), R4
JAL (R4)
// Set m->sched.sp = SP, so that if a panic happens
@ -559,10 +569,24 @@ havem:
MOVV savedsp-24(SP), R2 // must match frame size
MOVV R2, (g_sched+gobuf_sp)(g)
// If the m on entry was nil, we called needm above to borrow an m
// for the duration of the call. Since the call is over, return it with dropm.
// If the m on entry was nil, we called needm above to borrow an m,
// 1. for the duration of the call on non-pthread platforms,
// 2. or the duration of the C thread alive on pthread platforms.
// If the m on entry wasn't nil,
// 1. the thread might be a Go thread,
// 2. or it's wasn't the first call from a C thread on pthread platforms,
// since the we skip dropm to resue the m in the first call.
MOVV savedm-8(SP), R3
BNE R3, droppedm
// Skip dropm to reuse it in the next call, when a pthread key has been created.
MOVV _cgo_pthread_key_created(SB), R3
// It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
BEQ R3, dropm
MOVV (R3), R3
BNE R3, droppedm
dropm:
MOVV $runtime·dropm(SB), R4
JAL (R4)
droppedm:

32
src/runtime/asm_mipsx.s
View File

@ -459,13 +459,23 @@ g0:
TEXT ·cgocallback(SB),NOSPLIT,$12-12
NO_LOCAL_POINTERS
// Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
// It is used to dropm while thread is exiting.
MOVW fn+0(FP), R5
BNE R5, loadg
// Restore the g from frame.
MOVW frame+4(FP), g
JMP dropm
loadg:
// Load m and g from thread-local storage.
MOVB runtime·iscgo(SB), R1
BEQ R1, nocgo
JAL runtime·load_g(SB)
nocgo:
// If g is nil, Go did not create the current thread.
// If g is nil, Go did not create the current thread,
// or if this thread never called into Go on pthread platforms.
// Call needm to obtain one for temporary use.
// In this case, we're running on the thread stack, so there's
// lots of space, but the linker doesn't know. Hide the call from
@ -478,7 +488,7 @@ nocgo:
needm:
MOVW g, savedm-4(SP) // g is zero, so is m.
MOVW $runtime·needm(SB), R4
MOVW $runtime·needAndBindM(SB), R4
JAL (R4)
// Set m->sched.sp = SP, so that if a panic happens
@ -549,10 +559,24 @@ havem:
MOVW savedsp-12(SP), R2 // must match frame size
MOVW R2, (g_sched+gobuf_sp)(g)
// If the m on entry was nil, we called needm above to borrow an m
// for the duration of the call. Since the call is over, return it with dropm.
// If the m on entry was nil, we called needm above to borrow an m,
// 1. for the duration of the call on non-pthread platforms,
// 2. or the duration of the C thread alive on pthread platforms.
// If the m on entry wasn't nil,
// 1. the thread might be a Go thread,
// 2. or it's wasn't the first call from a C thread on pthread platforms,
// since the we skip dropm to resue the m in the first call.
MOVW savedm-4(SP), R3
BNE R3, droppedm
// Skip dropm to reuse it in the next call, when a pthread key has been created.
MOVW _cgo_pthread_key_created(SB), R3
// It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
BEQ R3, dropm
MOVW (R3), R3
BNE R3, droppedm
dropm:
MOVW $runtime·dropm(SB), R4
JAL (R4)
droppedm:

35
src/runtime/asm_ppc64x.s
View File

@ -628,6 +628,16 @@ g0:
TEXT ·cgocallback(SB),NOSPLIT,$24-24
NO_LOCAL_POINTERS
// Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
// It is used to dropm while thread is exiting.
MOVD fn+0(FP), R5
CMP R5, $0
BNE loadg
// Restore the g from frame.
MOVD frame+8(FP), g
BR dropm
loadg:
// Load m and g from thread-local storage.
MOVBZ runtime·iscgo(SB), R3
CMP R3, $0
@ -635,7 +645,8 @@ TEXT ·cgocallback(SB),NOSPLIT,$24-24
BL runtime·load_g(SB)
nocgo:
// If g is nil, Go did not create the current thread.
// If g is nil, Go did not create the current thread,
// or if this thread never called into Go on pthread platforms.
// Call needm to obtain one for temporary use.
// In this case, we're running on the thread stack, so there's
// lots of space, but the linker doesn't know. Hide the call from
@ -649,7 +660,7 @@ nocgo:
needm:
MOVD g, savedm-8(SP) // g is zero, so is m.
MOVD $runtime·needm(SB), R12
MOVD $runtime·needAndBindM(SB), R12
MOVD R12, CTR
BL (CTR)
@ -724,11 +735,27 @@ havem:
MOVD savedsp-24(SP), R4 // must match frame size
MOVD R4, (g_sched+gobuf_sp)(g)
// If the m on entry was nil, we called needm above to borrow an m
// for the duration of the call. Since the call is over, return it with dropm.
// If the m on entry was nil, we called needm above to borrow an m,
// 1. for the duration of the call on non-pthread platforms,
// 2. or the duration of the C thread alive on pthread platforms.
// If the m on entry wasn't nil,
// 1. the thread might be a Go thread,
// 2. or it's wasn't the first call from a C thread on pthread platforms,
// since the we skip dropm to resue the m in the first call.
MOVD savedm-8(SP), R6
CMP R6, $0
BNE droppedm
// Skip dropm to reuse it in the next call, when a pthread key has been created.
MOVD _cgo_pthread_key_created(SB), R6
// It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
CMP R6, $0
BEQ dropm
MOVD (R6), R6
CMP R6, $0
BNE droppedm
dropm:
MOVD $runtime·dropm(SB), R12
MOVD R12, CTR
BL (CTR)

32
src/runtime/asm_riscv64.s
View File

@ -519,13 +519,23 @@ TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
TEXT ·cgocallback(SB),NOSPLIT,$24-24
NO_LOCAL_POINTERS
// Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
// It is used to dropm while thread is exiting.
MOV fn+0(FP), X7
BNE ZERO, X7, loadg
// Restore the g from frame.
MOV frame+8(FP), g
JMP dropm
loadg:
// Load m and g from thread-local storage.
MOVBU runtime·iscgo(SB), X5
BEQ ZERO, X5, nocgo
CALL runtime·load_g(SB)
nocgo:
// If g is nil, Go did not create the current thread.
// If g is nil, Go did not create the current thread,
// or if this thread never called into Go on pthread platforms.
// Call needm to obtain one for temporary use.
// In this case, we're running on the thread stack, so there's
// lots of space, but the linker doesn't know. Hide the call from
@ -538,7 +548,7 @@ nocgo:
needm:
MOV g, savedm-8(SP) // g is zero, so is m.
MOV $runtime·needm(SB), X6
MOV $runtime·needAndBindM(SB), X6
JALR RA, X6
// Set m->sched.sp = SP, so that if a panic happens
@ -609,10 +619,24 @@ havem:
MOV savedsp-24(SP), X6 // must match frame size
MOV X6, (g_sched+gobuf_sp)(g)
// If the m on entry was nil, we called needm above to borrow an m
// for the duration of the call. Since the call is over, return it with dropm.
// If the m on entry was nil, we called needm above to borrow an m,
// 1. for the duration of the call on non-pthread platforms,
// 2. or the duration of the C thread alive on pthread platforms.
// If the m on entry wasn't nil,
// 1. the thread might be a Go thread,
// 2. or it's wasn't the first call from a C thread on pthread platforms,
// since the we skip dropm to resue the m in the first call.
MOV savedm-8(SP), X5
BNE ZERO, X5, droppedm
// Skip dropm to reuse it in the next call, when a pthread key has been created.
MOV _cgo_pthread_key_created(SB), X5
// It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
BEQ ZERO, X5, dropm
MOV (X5), X5
BNE ZERO, X5, droppedm
dropm:
MOV $runtime·dropm(SB), X6
JALR RA, X6
droppedm:

32
src/runtime/asm_s390x.s
View File

@ -564,13 +564,23 @@ g0:
TEXT ·cgocallback(SB),NOSPLIT,$24-24
NO_LOCAL_POINTERS
// Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
// It is used to dropm while thread is exiting.
MOVD fn+0(FP), R1
CMPBNE R1, $0, loadg
// Restore the g from frame.
MOVD frame+8(FP), g
BR dropm
loadg:
// Load m and g from thread-local storage.
MOVB runtime·iscgo(SB), R3
CMPBEQ R3, $0, nocgo
BL runtime·load_g(SB)
nocgo:
// If g is nil, Go did not create the current thread.
// If g is nil, Go did not create the current thread,
// or if this thread never called into Go on pthread platforms.
// Call needm to obtain one for temporary use.
// In this case, we're running on the thread stack, so there's
// lots of space, but the linker doesn't know. Hide the call from
@ -583,7 +593,7 @@ nocgo:
needm:
MOVD g, savedm-8(SP) // g is zero, so is m.
MOVD $runtime·needm(SB), R3
MOVD $runtime·needAndBindM(SB), R3
BL (R3)
// Set m->sched.sp = SP, so that if a panic happens
@ -654,10 +664,24 @@ havem:
MOVD savedsp-24(SP), R4 // must match frame size
MOVD R4, (g_sched+gobuf_sp)(g)
// If the m on entry was nil, we called needm above to borrow an m
// for the duration of the call. Since the call is over, return it with dropm.
// If the m on entry was nil, we called needm above to borrow an m,
// 1. for the duration of the call on non-pthread platforms,
// 2. or the duration of the C thread alive on pthread platforms.
// If the m on entry wasn't nil,
// 1. the thread might be a Go thread,
// 2. or it's wasn't the first call from a C thread on pthread platforms,
// since the we skip dropm to resue the m in the first call.
MOVD savedm-8(SP), R6
CMPBNE R6, $0, droppedm
// Skip dropm to reuse it in the next call, when a pthread key has been created.
MOVD _cgo_pthread_key_created(SB), R6
// It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
CMPBEQ R6, $0, dropm
MOVD (R6), R6
CMPBNE R6, $0, droppedm
dropm:
MOVD $runtime·dropm(SB), R3
BL (R3)
droppedm:

7
src/runtime/cgo.go
View File

@ -17,6 +17,8 @@ import "unsafe"
//go:linkname _cgo_callers _cgo_callers
//go:linkname _cgo_set_context_function _cgo_set_context_function
//go:linkname _cgo_yield _cgo_yield
//go:linkname _cgo_pthread_key_created _cgo_pthread_key_created
//go:linkname _cgo_bindm _cgo_bindm
var (
_cgo_init unsafe.Pointer
@ -26,11 +28,16 @@ var (
_cgo_callers unsafe.Pointer
_cgo_set_context_function unsafe.Pointer
_cgo_yield unsafe.Pointer
_cgo_pthread_key_created unsafe.Pointer
_cgo_bindm unsafe.Pointer
)
// iscgo is set to true by the runtime/cgo package
var iscgo bool
// set_crosscall2 is set by the runtime/cgo package
var set_crosscall2 func()
// cgoHasExtraM is set on startup when an extra M is created for cgo.
// The extra M must be created before any C/C++ code calls cgocallback.
var cgoHasExtraM bool

8
src/runtime/cgo/asm_386.s
View File

@ -4,6 +4,14 @@
#include "textflag.h"
// Set the x_crosscall2_ptr C function pointer variable point to crosscall2.
// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2
TEXT ·set_crosscall2(SB),NOSPLIT,$0-0
MOVL _crosscall2_ptr(SB), AX
MOVL $crosscall2(SB), BX
MOVL BX, (AX)
RET
// Called by C code generated by cmd/cgo.
// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr)
// Saves C callee-saved registers and calls cgocallback with three arguments.

8
src/runtime/cgo/asm_amd64.s
View File

@ -5,6 +5,14 @@
#include "textflag.h"
#include "abi_amd64.h"
// Set the x_crosscall2_ptr C function pointer variable point to crosscall2.
// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2
TEXT ·set_crosscall2(SB),NOSPLIT,$0-0
MOVQ _crosscall2_ptr(SB), AX
MOVQ $crosscall2(SB), BX
MOVQ BX, (AX)
RET
// Called by C code generated by cmd/cgo.
// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr)
// Saves C callee-saved registers and calls cgocallback with three arguments.

8
src/runtime/cgo/asm_arm.s
View File

@ -4,6 +4,14 @@
#include "textflag.h"
// Set the x_crosscall2_ptr C function pointer variable point to crosscall2.
// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2
TEXT ·set_crosscall2(SB),NOSPLIT,$0-0
MOVW _crosscall2_ptr(SB), R1
MOVW $crosscall2(SB), R2
MOVW R2, (R1)
RET
// Called by C code generated by cmd/cgo.
// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr)
// Saves C callee-saved registers and calls cgocallback with three arguments.

8
src/runtime/cgo/asm_arm64.s
View File

@ -5,6 +5,14 @@
#include "textflag.h"
#include "abi_arm64.h"
// Set the x_crosscall2_ptr C function pointer variable point to crosscall2.
// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2
TEXT ·set_crosscall2(SB),NOSPLIT,$0-0
MOVD _crosscall2_ptr(SB), R1
MOVD $crosscall2(SB), R2
MOVD R2, (R1)
RET
// Called by C code generated by cmd/cgo.
// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr)
// Saves C callee-saved registers and calls cgocallback with three arguments.

8
src/runtime/cgo/asm_loong64.s
View File

@ -4,6 +4,14 @@
#include "textflag.h"
// Set the x_crosscall2_ptr C function pointer variable point to crosscall2.
// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2
TEXT ·set_crosscall2(SB),NOSPLIT,$0-0
MOVV _crosscall2_ptr(SB), R5
MOVV $crosscall2(SB), R6
MOVV R6, (R5)
RET
// Called by C code generated by cmd/cgo.
// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr)
// Saves C callee-saved registers and calls cgocallback with three arguments.

8
src/runtime/cgo/asm_mips64x.s
View File

@ -6,6 +6,14 @@
#include "textflag.h"
// Set the x_crosscall2_ptr C function pointer variable point to crosscall2.
// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2
TEXT ·set_crosscall2(SB),NOSPLIT,$0-0
MOVV _crosscall2_ptr(SB), R5
MOVV $crosscall2(SB), R6
MOVV R6, (R5)
RET
// Called by C code generated by cmd/cgo.
// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr)
// Saves C callee-saved registers and calls cgocallback with three arguments.

8
src/runtime/cgo/asm_mipsx.s
View File

@ -6,6 +6,14 @@
#include "textflag.h"
// Set the x_crosscall2_ptr C function pointer variable point to crosscall2.
// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2
TEXT ·set_crosscall2(SB),NOSPLIT,$0-0
MOVW _crosscall2_ptr(SB), R5
MOVW $crosscall2(SB), R6
MOVW R6, (R5)
RET
// Called by C code generated by cmd/cgo.
// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr)
// Saves C callee-saved registers and calls cgocallback with three arguments.

24
src/runtime/cgo/asm_ppc64x.s
View File

@ -7,6 +7,26 @@
#include "textflag.h"
#include "asm_ppc64x.h"
// Set the x_crosscall2_ptr C function pointer variable point to crosscall2.
// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2
TEXT ·set_crosscall2(SB),NOSPLIT,$0-0
MOVD _crosscall2_ptr(SB), R5
#ifdef GOARCH_ppc64
MOVD $_crosscall2<>(SB), R6
#else
MOVD $crosscall2(SB), R6
#endif
MOVD R6, (R5)
RET
#ifdef GO_PPC64X_HAS_FUNCDESC
// _crosscall2<> is a function descriptor to the real crosscall2.
DATA _crosscall2<>+0(SB)/8, $crosscall2(SB)
DATA _crosscall2<>+8(SB)/8, $TOC(SB)
DATA _crosscall2<>+16(SB)/8, $0
GLOBL _crosscall2<>(SB), NOPTR, $24
#endif
// Called by C code generated by cmd/cgo.
// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr)
// Saves C callee-saved registers and calls cgocallback with three arguments.
@ -32,8 +52,12 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0
#ifdef GO_PPC64X_HAS_FUNCDESC
// Load the real entry address from the first slot of the function descriptor.
// The first argument fn might be null, that means dropm in pthread key destructor.
CMP R3, $0
BEQ nil_fn
MOVD 8(R3), R2
MOVD (R3), R3
nil_fn:
#endif
MOVD R3, FIXED_FRAME+0(R1) // fn unsafe.Pointer
MOVD R4, FIXED_FRAME+8(R1) // a unsafe.Pointer

8
src/runtime/cgo/asm_riscv64.s
View File

@ -4,6 +4,14 @@
#include "textflag.h"
// Set the x_crosscall2_ptr C function pointer variable point to crosscall2.
// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2
TEXT ·set_crosscall2(SB),NOSPLIT,$0-0
MOV _crosscall2_ptr(SB), X7
MOV $crosscall2(SB), X8
MOV X8, (X7)
RET
// Called by C code generated by cmd/cgo.
// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr)
// Saves C callee-saved registers and calls cgocallback with three arguments.

8
src/runtime/cgo/asm_s390x.s
View File

@ -4,6 +4,14 @@
#include "textflag.h"
// Set the x_crosscall2_ptr C function pointer variable point to crosscall2.
// It's such a pointer chain: _crosscall2_ptr -> x_crosscall2_ptr -> crosscall2
TEXT ·set_crosscall2(SB),NOSPLIT,$0-0
MOVD _crosscall2_ptr(SB), R1
MOVD $crosscall2(SB), R2
MOVD R2, (R1)
RET
// Called by C code generated by cmd/cgo.
// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr)
// Saves C callee-saved registers and calls cgocallback with three arguments.

3
src/runtime/cgo/asm_wasm.s
View File

@ -4,5 +4,8 @@
#include "textflag.h"
TEXT ·set_crosscall2(SB),NOSPLIT,$0-0
UNDEF
TEXT crosscall2(SB), NOSPLIT, $0
UNDEF

36
src/runtime/cgo/callbacks.go
View File

@ -71,6 +71,42 @@ var _cgo_thread_start = &x_cgo_thread_start
var x_cgo_sys_thread_create byte
var _cgo_sys_thread_create = &x_cgo_sys_thread_create
// Indicates whether a dummy thread key has been created or not.
//
// When calling go exported function from C, we register a destructor
// callback, for a dummy thread key, by using pthread_key_create.
//go:cgo_import_static x_cgo_pthread_key_created
//go:linkname x_cgo_pthread_key_created x_cgo_pthread_key_created
//go:linkname _cgo_pthread_key_created _cgo_pthread_key_created
var x_cgo_pthread_key_created byte
var _cgo_pthread_key_created = &x_cgo_pthread_key_created
// Export crosscall2 to a c function pointer variable.
// Used to dropm in pthread key destructor, while C thread is exiting.
//go:cgo_import_static x_crosscall2_ptr
//go:linkname x_crosscall2_ptr x_crosscall2_ptr
//go:linkname _crosscall2_ptr _crosscall2_ptr
var x_crosscall2_ptr byte
var _crosscall2_ptr = &x_crosscall2_ptr
// Set the x_crosscall2_ptr C function pointer variable point to crosscall2.
// It's for the runtime package to call at init time.
func set_crosscall2()
//go:linkname _set_crosscall2 runtime.set_crosscall2
var _set_crosscall2 = set_crosscall2
// Store the g into the thread-specific value.
// So that pthread_key_destructor will dropm when the thread is exiting.
//go:cgo_import_static x_cgo_bindm
//go:linkname x_cgo_bindm x_cgo_bindm
//go:linkname _cgo_bindm _cgo_bindm
var x_cgo_bindm byte
var _cgo_bindm = &x_cgo_bindm
// Notifies that the runtime has been initialized.
//
// We currently block at every CGO entry point (via _cgo_wait_runtime_init_done)

35
src/runtime/cgo/gcc_libinit.c
View File

@ -17,6 +17,14 @@ static pthread_cond_t runtime_init_cond = PTHREAD_COND_INITIALIZER;
static pthread_mutex_t runtime_init_mu = PTHREAD_MUTEX_INITIALIZER;
static int runtime_init_done;
// pthread_g is a pthread specific key, for storing the g that binded to the C thread.
// The registered pthread_key_destructor will dropm, when the pthread-specified value g is not NULL,
// while a C thread is exiting.
static pthread_key_t pthread_g;
static void pthread_key_destructor(void* g);
uintptr_t x_cgo_pthread_key_created;
void (*x_crosscall2_ptr)(void (*fn)(void *), void *, int, size_t);
// The context function, used when tracing back C calls into Go.
static void (*cgo_context_function)(struct context_arg*);
@ -39,6 +47,12 @@ _cgo_wait_runtime_init_done(void) {
pthread_cond_wait(&runtime_init_cond, &runtime_init_mu);
}
// The key and x_cgo_pthread_key_created are for the whole program,
// whereas the specific and destructor is per thread.
if (x_cgo_pthread_key_created == 0 && pthread_key_create(&pthread_g, pthread_key_destructor) == 0) {
x_cgo_pthread_key_created = 1;
}
// TODO(iant): For the case of a new C thread calling into Go, such
// as when using -buildmode=c-archive, we know that Go runtime
// initialization is complete but we do not know that all Go init
@ -61,6 +75,16 @@ _cgo_wait_runtime_init_done(void) {
return 0;
}
// Store the g into a thread-specific value associated with the pthread key pthread_g.
// And pthread_key_destructor will dropm when the thread is exiting.
void x_cgo_bindm(void* g) {
// We assume this will always succeed, otherwise, there might be extra M leaking,
// when a C thread exits after a cgo call.
// We only invoke this function once per thread in runtime.needAndBindM,
// and the next calls just reuse the bound m.
pthread_setspecific(pthread_g, g);
}
void
x_cgo_notify_runtime_init_done(void* dummy __attribute__ ((unused))) {
pthread_mutex_lock(&runtime_init_mu);
@ -110,3 +134,14 @@ _cgo_try_pthread_create(pthread_t* thread, const pthread_attr_t* attr, void* (*p
}
return EAGAIN;
}
static void
pthread_key_destructor(void* g) {
if (x_crosscall2_ptr != NULL) {
// fn == NULL means dropm.
// We restore g by using the stored g, before dropm in runtime.cgocallback,
// since the g stored in the TLS by Go might be cleared in some platforms,
// before this destructor invoked.
x_crosscall2_ptr(NULL, g, 0, 0);
}
}

9
src/runtime/cgo/gcc_libinit_windows.c
View File

@ -30,6 +30,9 @@ static CRITICAL_SECTION runtime_init_cs;
static HANDLE runtime_init_wait;
static int runtime_init_done;
uintptr_t x_cgo_pthread_key_created;
void (*x_crosscall2_ptr)(void (*fn)(void *), void *, int, size_t);
// Pre-initialize the runtime synchronization objects
void
_cgo_preinit_init() {
@ -91,6 +94,12 @@ _cgo_wait_runtime_init_done(void) {
return 0;
}
// Should not be used since x_cgo_pthread_key_created will always be zero.
void x_cgo_bindm(void* dummy) {
fprintf(stderr, "unexpected cgo_bindm on Windows\n");
abort();
}
void
x_cgo_notify_runtime_init_done(void* dummy) {
_cgo_maybe_run_preinit();

5
src/runtime/cgo/libcgo.h
View File

@ -51,6 +51,11 @@ extern void (*_cgo_thread_start)(ThreadStart *ts);
*/
extern void (*_cgo_sys_thread_create)(void* (*func)(void*), void* arg);
/*
* Indicates whether a dummy pthread per-thread variable is allocated.
*/
extern uintptr_t *_cgo_pthread_key_created;
/*
* Creates the new operating system thread (OS, arch dependent).
*/

6
src/runtime/cgocall.go
View File

@ -229,6 +229,9 @@ func cgocallbackg(fn, frame unsafe.Pointer, ctxt uintptr) {
savedpc := gp.syscallpc
exitsyscall() // coming out of cgo call
gp.m.incgo = false
if gp.m.isextra {
gp.m.isExtraInC = false
}
osPreemptExtExit(gp.m)
@ -239,6 +242,9 @@ func cgocallbackg(fn, frame unsafe.Pointer, ctxt uintptr) {
// This is enforced by checking incgo in the schedule function.
gp.m.incgo = true
if gp.m.isextra {
gp.m.isExtraInC = true
}
if gp.m != checkm {
throw("m changed unexpectedly in cgocallbackg")

13
src/runtime/crash_cgo_test.go
View File

@ -771,3 +771,16 @@ func TestCgoSigfwd(t *testing.T) {
t.Fatalf("expected %q, but got:\n%s", want, got)
}
}
func TestEnsureBindM(t *testing.T) {
t.Parallel()
switch runtime.GOOS {
case "windows", "plan9":
t.Skipf("skipping bindm test on %s", runtime.GOOS)
}
got := runTestProg(t, "testprogcgo", "EnsureBindM")
want := "OK\n"
if got != want {
t.Errorf("expected %q, got %v", want, got)
}
}

92
src/runtime/proc.go
View File

@ -210,6 +210,10 @@ func main() {
main_init_done = make(chan bool)
if iscgo {
if _cgo_pthread_key_created == nil {
throw("_cgo_pthread_key_created missing")
}
if _cgo_thread_start == nil {
throw("_cgo_thread_start missing")
}
@ -224,6 +228,13 @@ func main() {
if _cgo_notify_runtime_init_done == nil {
throw("_cgo_notify_runtime_init_done missing")
}
// Set the x_crosscall2_ptr C function pointer variable point to crosscall2.
if set_crosscall2 == nil {
throw("set_crosscall2 missing")
}
set_crosscall2()
// Start the template thread in case we enter Go from
// a C-created thread and need to create a new thread.
startTemplateThread()
@ -1870,8 +1881,9 @@ func allocm(pp *p, fn func(), id int64) *m {
// pressed into service as the scheduling stack and current
// goroutine for the duration of the cgo callback.
//
// When the callback is done with the m, it calls dropm to
// put the m back on the list.
// It calls dropm to put the m back on the list,
// 1. when the callback is done with the m in non-pthread platforms,
// 2. or when the C thread exiting on pthread platforms.
//
//go:nosplit
func needm() {
@ -1933,6 +1945,11 @@ func needm() {
gp.stack.lo = getcallersp() - 32*1024
gp.stackguard0 = gp.stack.lo + _StackGuard
// Should mark we are already in Go now.
// Otherwise, we may call needm again when we get a signal, before cgocallbackg1,
// which means the extram list may be empty, that will cause a deadlock.
mp.isExtraInC = false
// Initialize this thread to use the m.
asminit()
minit()
@ -1942,6 +1959,17 @@ func needm() {
sched.ngsys.Add(-1)
}
// Acquire an extra m and bind it to the C thread when a pthread key has been created.
//
//go:nosplit
func needAndBindM() {
needm()
if _cgo_pthread_key_created != nil && *(*uintptr)(_cgo_pthread_key_created) != 0 {
cgoBindM()
}
}
// newextram allocates m's and puts them on the extra list.
// It is called with a working local m, so that it can do things
// like call schedlock and allocate.
@ -1986,6 +2014,8 @@ func oneNewExtraM() {
gp.m = mp
mp.curg = gp
mp.isextra = true
// mark we are in C by default.
mp.isExtraInC = true
mp.lockedInt++
mp.lockedg.set(gp)
gp.lockedm.set(mp)
@ -2018,9 +2048,11 @@ func oneNewExtraM() {
unlockextra(mp)
}
// dropm puts the current m back onto the extra list.
//
// 1. On systems without pthreads, like Windows
// dropm is called when a cgo callback has called needm but is now
// done with the callback and returning back into the non-Go thread.
// It puts the current m back onto the extra list.
//
// The main expense here is the call to signalstack to release the
// m's signal stack, and then the call to needm on the next callback
@ -2032,15 +2064,18 @@ func oneNewExtraM() {
// call. These should typically not be scheduling operations, just a few
// atomics, so the cost should be small.
//
// TODO(rsc): An alternative would be to allocate a dummy pthread per-thread
// variable using pthread_key_create. Unlike the pthread keys we already use
// on OS X, this dummy key would never be read by Go code. It would exist
// only so that we could register at thread-exit-time destructor.
// That destructor would put the m back onto the extra list.
// This is purely a performance optimization. The current version,
// in which dropm happens on each cgo call, is still correct too.
// We may have to keep the current version on systems with cgo
// but without pthreads, like Windows.
// 2. On systems with pthreads
// dropm is called while a non-Go thread is exiting.
// We allocate a pthread per-thread variable using pthread_key_create,
// to register a thread-exit-time destructor.
// And store the g into a thread-specific value associated with the pthread key,
// when first return back to C.
// So that the destructor would invoke dropm while the non-Go thread is exiting.
// This is much faster since it avoids expensive signal-related syscalls.
//
// NOTE: this always runs without a P, so, nowritebarrierrec required.
//
//go:nowritebarrierrec
func dropm() {
// Clear m and g, and return m to the extra list.
// After the call to setg we can only call nosplit functions
@ -2072,6 +2107,39 @@ func dropm() {
msigrestore(sigmask)
}
// bindm store the g0 of the current m into a thread-specific value.
//
// We allocate a pthread per-thread variable using pthread_key_create,
// to register a thread-exit-time destructor.
// We are here setting the thread-specific value of the pthread key, to enable the destructor.
// So that the pthread_key_destructor would dropm while the C thread is exiting.
//
// And the saved g will be used in pthread_key_destructor,
// since the g stored in the TLS by Go might be cleared in some platforms,
// before the destructor invoked, so, we restore g by the stored g, before dropm.
//
// We store g0 instead of m, to make the assembly code simpler,
// since we need to restore g0 in runtime.cgocallback.
//
// On systems without pthreads, like Windows, bindm shouldn't be used.
//
// NOTE: this always runs without a P, so, nowritebarrierrec required.
//
//go:nosplit
//go:nowritebarrierrec
func cgoBindM() {
if GOOS == "windows" || GOOS == "plan9" {
fatal("bindm in unexpected GOOS")
}
g := getg()
if g.m.g0 != g {
fatal("the current g is not g0")
}
if _cgo_bindm != nil {
asmcgocall(_cgo_bindm, unsafe.Pointer(g))
}
}
// A helper function for EnsureDropM.
func getm() uintptr {
return uintptr(unsafe.Pointer(getg().m))

1
src/runtime/runtime2.go
View File

@ -561,6 +561,7 @@ type m struct {
printlock int8
incgo bool // m is executing a cgo call
isextra bool // m is an extra m
isExtraInC bool // m is an extra m that is not executing Go code
freeWait atomic.Uint32 // Whether it is safe to free g0 and delete m (one of freeMRef, freeMStack, freeMWait)
fastrand uint64
needextram bool

16
src/runtime/signal_unix.go
View File

@ -435,7 +435,7 @@ func sigtrampgo(sig uint32, info *siginfo, ctx unsafe.Pointer) {
c := &sigctxt{info, ctx}
gp := sigFetchG(c)
setg(gp)
if gp == nil {
if gp == nil || (gp.m != nil && gp.m.isExtraInC) {
if sig == _SIGPROF {
// Some platforms (Linux) have per-thread timers, which we use in
// combination with the process-wide timer. Avoid double-counting.
@ -458,7 +458,18 @@ func sigtrampgo(sig uint32, info *siginfo, ctx unsafe.Pointer) {
return
}
c.fixsigcode(sig)
// Set g to nil here and badsignal will use g0 by needm.
// TODO: reuse the current m here by using the gsignal and adjustSignalStack,
// since the current g maybe a normal goroutine and actually running on the signal stack,
// it may hit stack split that is not expected here.
if gp != nil {
setg(nil)
}
badsignal(uintptr(sig), c)
// Restore g
if gp != nil {
setg(gp)
}
return
}
@ -1121,8 +1132,9 @@ func sigfwdgo(sig uint32, info *siginfo, ctx unsafe.Pointer) bool {
// (1) we weren't in VDSO page,
// (2) we were in a goroutine (i.e., m.curg != nil), and
// (3) we weren't in CGO.
// (4) we weren't in dropped extra m.
gp := sigFetchG(c)
if gp != nil && gp.m != nil && gp.m.curg != nil && !gp.m.incgo {
if gp != nil && gp.m != nil && gp.m.curg != nil && !gp.m.isExtraInC && !gp.m.incgo {
return false
}

3
src/runtime/stubs.go
View File

@ -237,6 +237,9 @@ func noEscapePtr[T any](p *T) *T {
// cgocallback is not called from Go, only from crosscall2.
// This in turn calls cgocallbackg, which is where we'll find
// pointer-declared arguments.
//
// When fn is nil (frame is saved g), call dropm instead,
// this is used when the C thread is exiting.
func cgocallback(fn, frame, ctxt uintptr)
func gogo(buf *gobuf)

89
src/runtime/testdata/testprogcgo/bindm.go vendored Normal file
View File

@ -0,0 +1,89 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !plan9 && !windows
// Test that callbacks from C to Go in the same C-thread always get the same m.
// Make sure the extra M bind to the C-thread.
package main
/*
#include <stdint.h>
#include <pthread.h>
#include <unistd.h>
extern void GoCheckBindM();
#define CTHREADS 2
#define CHECKCALLS 100
static void* checkBindMThread(void* thread) {
int i;
for (i = 0; i < CHECKCALLS; i++) {
GoCheckBindM((uintptr_t)thread);
usleep(1);
}
return NULL;
}
static void CheckBindM() {
int i;
pthread_t s[CTHREADS];
for (i = 0; i < CTHREADS; i++) {
pthread_create(&s[i], NULL, checkBindMThread, &s[i]);
}
for (i = 0; i < CTHREADS; i++) {
pthread_join(s[i], NULL);
}
}
*/
import "C"
import (
"fmt"
"os"
"runtime"
"sync"
"sync/atomic"
)
var (
mutex = sync.Mutex{}
cThreadToM = map[uintptr]uintptr{}
started = atomic.Uint32{}
)
// same as CTHREADS in C, make sure all the C threads are actually started.
const cThreadNum = 2
func init() {
register("EnsureBindM", EnsureBindM)
}
//export GoCheckBindM
func GoCheckBindM(thread uintptr) {
// Wait all threads start
if started.Load() != cThreadNum {
// Only once for each thread, since it will wait all threads start.
started.Add(1)
for started.Load() < cThreadNum {
runtime.Gosched()
}
}
m := runtime_getm_for_test()
mutex.Lock()
defer mutex.Unlock()
if savedM, ok := cThreadToM[thread]; ok && savedM != m {
fmt.Printf("m == %x want %x\n", m, savedM)
os.Exit(1)
}
cThreadToM[thread] = m
}
func EnsureBindM() {
C.CheckBindM()
fmt.Println("OK")
}