2008-06-30 12:50:36 -06:00
|
|
|
// Copyright 2009 The Go Authors. All rights reserved.
|
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
2011-12-19 13:51:13 -07:00
|
|
|
#include "zasm_GOOS_GOARCH.h"
|
2008-06-30 12:50:36 -06:00
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT _rt0_amd64(SB),7,$-8
|
2008-06-30 12:50:36 -06:00
|
|
|
// copy arguments forward on an even stack
|
2013-03-06 13:03:04 -07:00
|
|
|
MOVQ DI, AX // argc
|
|
|
|
|
MOVQ SI, BX // argv
|
2008-06-30 12:50:36 -06:00
|
|
|
SUBQ $(4*8+7), SP // 2args 2auto
|
2010-04-09 15:15:15 -06:00
|
|
|
ANDQ $~15, SP
|
2008-06-30 12:50:36 -06:00
|
|
|
MOVQ AX, 16(SP)
|
|
|
|
|
MOVQ BX, 24(SP)
|
2011-12-07 06:53:17 -07:00
|
|
|
|
|
|
|
|
// create istack out of the given (operating system) stack.
|
2013-02-28 14:24:38 -07:00
|
|
|
// _cgo_init may update stackguard.
|
2011-12-07 06:53:17 -07:00
|
|
|
MOVQ $runtime·g0(SB), DI
|
2012-01-19 18:59:44 -07:00
|
|
|
LEAQ (-64*1024+104)(SP), BX
|
2011-12-07 06:53:17 -07:00
|
|
|
MOVQ BX, g_stackguard(DI)
|
|
|
|
|
MOVQ SP, g_stackbase(DI)
|
2008-06-30 12:50:36 -06:00
|
|
|
|
2013-03-12 11:47:44 -06:00
|
|
|
// find out information about the processor we're on
|
|
|
|
|
MOVQ $0, AX
|
|
|
|
|
CPUID
|
|
|
|
|
CMPQ AX, $0
|
|
|
|
|
JE nocpuinfo
|
|
|
|
|
MOVQ $1, AX
|
|
|
|
|
CPUID
|
|
|
|
|
MOVL CX, runtime·cpuid_ecx(SB)
|
|
|
|
|
MOVL DX, runtime·cpuid_edx(SB)
|
|
|
|
|
nocpuinfo:
|
|
|
|
|
|
2013-02-28 14:24:38 -07:00
|
|
|
// if there is an _cgo_init, call it.
|
|
|
|
|
MOVQ _cgo_init(SB), AX
|
2010-04-09 15:15:15 -06:00
|
|
|
TESTQ AX, AX
|
2010-08-04 18:50:22 -06:00
|
|
|
JZ needtls
|
2012-01-19 18:59:44 -07:00
|
|
|
// g0 already in DI
|
|
|
|
|
MOVQ DI, CX // Win64 uses CX for first parameter
|
2013-03-25 16:14:02 -06:00
|
|
|
MOVQ $setmg_gcc<>(SB), SI
|
2012-01-19 18:59:44 -07:00
|
|
|
CALL AX
|
2011-07-19 08:47:33 -06:00
|
|
|
CMPL runtime·iswindows(SB), $0
|
|
|
|
|
JEQ ok
|
2010-08-04 18:50:22 -06:00
|
|
|
|
|
|
|
|
needtls:
|
2012-08-31 11:21:13 -06:00
|
|
|
// skip TLS setup on Plan 9
|
|
|
|
|
CMPL runtime·isplan9(SB), $1
|
|
|
|
|
JEQ ok
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
LEAQ runtime·tls0(SB), DI
|
|
|
|
|
CALL runtime·settls(SB)
|
2010-08-04 18:50:22 -06:00
|
|
|
|
|
|
|
|
// store through it, to make sure it works
|
|
|
|
|
get_tls(BX)
|
|
|
|
|
MOVQ $0x123, g(BX)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
MOVQ runtime·tls0(SB), AX
|
2010-08-04 18:50:22 -06:00
|
|
|
CMPQ AX, $0x123
|
|
|
|
|
JEQ 2(PC)
|
|
|
|
|
MOVL AX, 0 // abort
|
|
|
|
|
ok:
|
|
|
|
|
// set the per-goroutine and per-mach "registers"
|
|
|
|
|
get_tls(BX)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
LEAQ runtime·g0(SB), CX
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ CX, g(BX)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
LEAQ runtime·m0(SB), AX
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ AX, m(BX)
|
|
|
|
|
|
|
|
|
|
// save m->g0 = g0
|
|
|
|
|
MOVQ CX, m_g0(AX)
|
2008-06-30 12:50:36 -06:00
|
|
|
|
2008-12-15 16:07:35 -07:00
|
|
|
CLD // convention is D is always left cleared
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
CALL runtime·check(SB)
|
2008-06-30 12:50:36 -06:00
|
|
|
|
|
|
|
|
MOVL 16(SP), AX // copy argc
|
|
|
|
|
MOVL AX, 0(SP)
|
|
|
|
|
MOVQ 24(SP), AX // copy argv
|
|
|
|
|
MOVQ AX, 8(SP)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
CALL runtime·args(SB)
|
|
|
|
|
CALL runtime·osinit(SB)
|
2013-03-12 11:47:44 -06:00
|
|
|
CALL runtime·hashinit(SB)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
CALL runtime·schedinit(SB)
|
2008-08-05 15:21:42 -06:00
|
|
|
|
2008-07-11 20:16:39 -06:00
|
|
|
// create a new goroutine to start program
|
2013-02-21 15:01:13 -07:00
|
|
|
PUSHQ $runtime·main·f(SB) // entry
|
2009-06-17 16:12:16 -06:00
|
|
|
PUSHQ $0 // arg size
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
CALL runtime·newproc(SB)
|
2008-07-11 20:16:39 -06:00
|
|
|
POPQ AX
|
|
|
|
|
POPQ AX
|
2008-12-04 09:30:54 -07:00
|
|
|
|
2008-09-22 14:47:59 -06:00
|
|
|
// start this M
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
CALL runtime·mstart(SB)
|
2008-06-30 12:50:36 -06:00
|
|
|
|
2012-03-08 12:03:56 -07:00
|
|
|
MOVL $0xf1, 0xf1 // crash
|
2008-06-30 12:50:36 -06:00
|
|
|
RET
|
|
|
|
|
|
2013-02-21 15:01:13 -07:00
|
|
|
DATA runtime·main·f+0(SB)/8,$runtime·main(SB)
|
|
|
|
|
GLOBL runtime·main·f(SB),8,$8
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·breakpoint(SB),7,$0
|
2008-07-11 20:16:39 -06:00
|
|
|
BYTE $0xcc
|
2008-06-30 12:50:36 -06:00
|
|
|
RET
|
|
|
|
|
|
2012-02-13 23:23:15 -07:00
|
|
|
TEXT runtime·asminit(SB),7,$0
|
|
|
|
|
// No per-thread init.
|
|
|
|
|
RET
|
|
|
|
|
|
2008-07-11 20:16:39 -06:00
|
|
|
/*
|
|
|
|
|
* go-routine
|
|
|
|
|
*/
|
2008-07-07 18:59:32 -06:00
|
|
|
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// void gosave(Gobuf*)
|
2009-06-17 16:12:16 -06:00
|
|
|
// save state in Gobuf; setjmp
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·gosave(SB), 7, $0
|
2008-07-11 20:16:39 -06:00
|
|
|
MOVQ 8(SP), AX // gobuf
|
2009-06-17 16:12:16 -06:00
|
|
|
LEAQ 8(SP), BX // caller's SP
|
|
|
|
|
MOVQ BX, gobuf_sp(AX)
|
|
|
|
|
MOVQ 0(SP), BX // caller's PC
|
|
|
|
|
MOVQ BX, gobuf_pc(AX)
|
2010-08-04 18:50:22 -06:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ g(CX), BX
|
|
|
|
|
MOVQ BX, gobuf_g(AX)
|
2008-06-30 12:50:36 -06:00
|
|
|
RET
|
|
|
|
|
|
2009-06-17 16:12:16 -06:00
|
|
|
// void gogo(Gobuf*, uintptr)
|
|
|
|
|
// restore state from Gobuf; longjmp
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·gogo(SB), 7, $0
|
2009-06-17 16:12:16 -06:00
|
|
|
MOVQ 16(SP), AX // return 2nd arg
|
|
|
|
|
MOVQ 8(SP), BX // gobuf
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ gobuf_g(BX), DX
|
|
|
|
|
MOVQ 0(DX), CX // make sure g != nil
|
|
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ DX, g(CX)
|
2009-06-17 16:12:16 -06:00
|
|
|
MOVQ gobuf_sp(BX), SP // restore SP
|
|
|
|
|
MOVQ gobuf_pc(BX), BX
|
|
|
|
|
JMP BX
|
|
|
|
|
|
2013-02-22 08:47:54 -07:00
|
|
|
// void gogocall(Gobuf*, void (*fn)(void), uintptr r0)
|
2009-06-17 16:12:16 -06:00
|
|
|
// restore state from Gobuf but then call fn.
|
|
|
|
|
// (call fn, returning to state in Gobuf)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·gogocall(SB), 7, $0
|
2013-02-22 08:47:54 -07:00
|
|
|
MOVQ 24(SP), DX // context
|
2009-06-17 16:12:16 -06:00
|
|
|
MOVQ 16(SP), AX // fn
|
|
|
|
|
MOVQ 8(SP), BX // gobuf
|
2013-02-22 08:47:54 -07:00
|
|
|
MOVQ gobuf_g(BX), DI
|
2010-08-04 18:50:22 -06:00
|
|
|
get_tls(CX)
|
2013-02-22 08:47:54 -07:00
|
|
|
MOVQ DI, g(CX)
|
|
|
|
|
MOVQ 0(DI), CX // make sure g != nil
|
2009-06-17 16:12:16 -06:00
|
|
|
MOVQ gobuf_sp(BX), SP // restore SP
|
|
|
|
|
MOVQ gobuf_pc(BX), BX
|
|
|
|
|
PUSHQ BX
|
|
|
|
|
JMP AX
|
|
|
|
|
POPQ BX // not reached
|
|
|
|
|
|
2013-02-21 15:01:13 -07:00
|
|
|
// void gogocallfn(Gobuf*, FuncVal*)
|
|
|
|
|
// restore state from Gobuf but then call fn.
|
|
|
|
|
// (call fn, returning to state in Gobuf)
|
|
|
|
|
TEXT runtime·gogocallfn(SB), 7, $0
|
2013-02-22 08:47:54 -07:00
|
|
|
MOVQ 16(SP), DX // fn
|
2013-02-21 15:01:13 -07:00
|
|
|
MOVQ 8(SP), BX // gobuf
|
2013-02-22 08:47:54 -07:00
|
|
|
MOVQ gobuf_g(BX), AX
|
2013-02-21 15:01:13 -07:00
|
|
|
get_tls(CX)
|
2013-02-22 08:47:54 -07:00
|
|
|
MOVQ AX, g(CX)
|
|
|
|
|
MOVQ 0(AX), CX // make sure g != nil
|
2013-02-21 15:01:13 -07:00
|
|
|
MOVQ gobuf_sp(BX), SP // restore SP
|
|
|
|
|
MOVQ gobuf_pc(BX), BX
|
|
|
|
|
PUSHQ BX
|
2013-02-22 08:47:54 -07:00
|
|
|
MOVQ 0(DX), BX
|
2013-02-21 15:01:13 -07:00
|
|
|
JMP BX
|
|
|
|
|
POPQ BX // not reached
|
|
|
|
|
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// void mcall(void (*fn)(G*))
|
|
|
|
|
// Switch to m->g0's stack, call fn(g).
|
runtime: stack split + garbage collection bug
The g->sched.sp saved stack pointer and the
g->stackbase and g->stackguard stack bounds
can change even while "the world is stopped",
because a goroutine has to call functions (and
therefore might split its stack) when exiting a
system call to check whether the world is stopped
(and if so, wait until the world continues).
That means the garbage collector cannot access
those values safely (without a race) for goroutines
executing system calls. Instead, save a consistent
triple in g->gcsp, g->gcstack, g->gcguard during
entersyscall and have the garbage collector refer
to those.
The old code was occasionally seeing (because of
the race) an sp and stk that did not correspond to
each other, so that stk - sp was not the number of
stack bytes following sp. In that case, if sp < stk
then the call scanblock(sp, stk - sp) scanned too
many bytes (anything between the two pointers,
which pointed into different allocation blocks).
If sp > stk then stk - sp wrapped around.
On 32-bit, stk - sp is a uintptr (uint32) converted
to int64 in the call to scanblock, so a large (~4G)
but positive number. Scanblock would try to scan
that many bytes and eventually fault accessing
unmapped memory. On 64-bit, stk - sp is a uintptr (uint64)
promoted to int64 in the call to scanblock, so a negative
number. Scanblock would not scan anything, possibly
causing in-use blocks to be freed.
In short, 32-bit platforms would have seen either
ineffective garbage collection or crashes during garbage
collection, while 64-bit platforms would have seen
either ineffective or incorrect garbage collection.
You can see the invalid arguments to scanblock in the
stack traces in issue 1620.
Fixes #1620.
Fixes #1746.
R=iant, r
CC=golang-dev
https://golang.org/cl/4437075
2011-04-27 21:21:12 -06:00
|
|
|
// Fn must never return. It should gogo(&g->sched)
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// to keep running g.
|
|
|
|
|
TEXT runtime·mcall(SB), 7, $0
|
|
|
|
|
MOVQ fn+0(FP), DI
|
|
|
|
|
|
|
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ g(CX), AX // save state in g->gobuf
|
|
|
|
|
MOVQ 0(SP), BX // caller's PC
|
|
|
|
|
MOVQ BX, (g_sched+gobuf_pc)(AX)
|
|
|
|
|
LEAQ 8(SP), BX // caller's SP
|
|
|
|
|
MOVQ BX, (g_sched+gobuf_sp)(AX)
|
|
|
|
|
MOVQ AX, (g_sched+gobuf_g)(AX)
|
|
|
|
|
|
|
|
|
|
// switch to m->g0 & its stack, call fn
|
|
|
|
|
MOVQ m(CX), BX
|
|
|
|
|
MOVQ m_g0(BX), SI
|
|
|
|
|
CMPQ SI, AX // if g == m->g0 call badmcall
|
|
|
|
|
JNE 2(PC)
|
|
|
|
|
CALL runtime·badmcall(SB)
|
|
|
|
|
MOVQ SI, g(CX) // g = m->g0
|
|
|
|
|
MOVQ (g_sched+gobuf_sp)(SI), SP // sp = m->g0->gobuf.sp
|
|
|
|
|
PUSHQ AX
|
|
|
|
|
CALL DI
|
|
|
|
|
POPQ AX
|
|
|
|
|
CALL runtime·badmcall2(SB)
|
|
|
|
|
RET
|
|
|
|
|
|
2008-07-12 12:30:53 -06:00
|
|
|
/*
|
|
|
|
|
* support for morestack
|
|
|
|
|
*/
|
|
|
|
|
|
2009-06-17 16:12:16 -06:00
|
|
|
// Called during function prolog when more stack is needed.
|
2010-08-04 18:50:22 -06:00
|
|
|
// Caller has already done get_tls(CX); MOVQ m(CX), BX.
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·morestack(SB),7,$0
|
2010-08-04 18:50:22 -06:00
|
|
|
// Cannot grow scheduler stack (m->g0).
|
|
|
|
|
MOVQ m_g0(BX), SI
|
|
|
|
|
CMPQ g(CX), SI
|
|
|
|
|
JNE 2(PC)
|
|
|
|
|
INT $3
|
2013-02-22 08:47:54 -07:00
|
|
|
|
|
|
|
|
MOVQ DX, m_cret(BX)
|
2010-08-04 18:50:22 -06:00
|
|
|
|
2009-06-17 16:12:16 -06:00
|
|
|
// Called from f.
|
|
|
|
|
// Set m->morebuf to f's caller.
|
|
|
|
|
MOVQ 8(SP), AX // f's caller's PC
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ AX, (m_morebuf+gobuf_pc)(BX)
|
2009-06-17 16:12:16 -06:00
|
|
|
LEAQ 16(SP), AX // f's caller's SP
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ AX, (m_morebuf+gobuf_sp)(BX)
|
2011-01-14 12:05:20 -07:00
|
|
|
MOVQ AX, m_moreargp(BX)
|
2010-08-04 18:50:22 -06:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ g(CX), SI
|
|
|
|
|
MOVQ SI, (m_morebuf+gobuf_g)(BX)
|
2009-06-17 16:12:16 -06:00
|
|
|
|
|
|
|
|
// Set m->morepc to f's PC.
|
|
|
|
|
MOVQ 0(SP), AX
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ AX, m_morepc(BX)
|
2009-06-17 16:12:16 -06:00
|
|
|
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// Call newstack on m->g0's stack.
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ m_g0(BX), BP
|
|
|
|
|
MOVQ BP, g(CX)
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ (g_sched+gobuf_sp)(BP), SP
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
CALL runtime·newstack(SB)
|
2009-06-17 16:12:16 -06:00
|
|
|
MOVQ $0, 0x1003 // crash if newstack returns
|
2009-07-08 19:16:09 -06:00
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
// Called from reflection library. Mimics morestack,
|
|
|
|
|
// reuses stack growth code to create a frame
|
|
|
|
|
// with the desired args running the desired function.
|
|
|
|
|
//
|
|
|
|
|
// func call(fn *byte, arg *byte, argsize uint32).
|
|
|
|
|
TEXT reflect·call(SB), 7, $0
|
2010-08-04 18:50:22 -06:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ m(CX), BX
|
|
|
|
|
|
2009-07-08 19:16:09 -06:00
|
|
|
// Save our caller's state as the PC and SP to
|
|
|
|
|
// restore when returning from f.
|
|
|
|
|
MOVQ 0(SP), AX // our caller's PC
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ AX, (m_morebuf+gobuf_pc)(BX)
|
2009-07-08 19:16:09 -06:00
|
|
|
LEAQ 8(SP), AX // our caller's SP
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ AX, (m_morebuf+gobuf_sp)(BX)
|
|
|
|
|
MOVQ g(CX), AX
|
|
|
|
|
MOVQ AX, (m_morebuf+gobuf_g)(BX)
|
2009-07-08 19:16:09 -06:00
|
|
|
|
|
|
|
|
// Set up morestack arguments to call f on a new stack.
|
2010-03-29 22:48:22 -06:00
|
|
|
// We set f's frame size to 1, as a hint to newstack
|
|
|
|
|
// that this is a call from reflect·call.
|
|
|
|
|
// If it turns out that f needs a larger frame than
|
|
|
|
|
// the default stack, f's usual stack growth prolog will
|
|
|
|
|
// allocate a new segment (and recopy the arguments).
|
2009-07-08 19:16:09 -06:00
|
|
|
MOVQ 8(SP), AX // fn
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ 16(SP), DX // arg frame
|
2009-07-08 19:16:09 -06:00
|
|
|
MOVL 24(SP), CX // arg size
|
|
|
|
|
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ AX, m_morepc(BX) // f's PC
|
2011-01-14 12:05:20 -07:00
|
|
|
MOVQ DX, m_moreargp(BX) // argument frame pointer
|
|
|
|
|
MOVL CX, m_moreargsize(BX) // f's argument size
|
|
|
|
|
MOVL $1, m_moreframesize(BX) // f's frame size
|
2009-07-08 19:16:09 -06:00
|
|
|
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// Call newstack on m->g0's stack.
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ m_g0(BX), BP
|
|
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ BP, g(CX)
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ (g_sched+gobuf_sp)(BP), SP
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
CALL runtime·newstack(SB)
|
2009-07-08 19:16:09 -06:00
|
|
|
MOVQ $0, 0x1103 // crash if newstack returns
|
2009-06-17 16:12:16 -06:00
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
// Return point when leaving stack.
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·lessstack(SB), 7, $0
|
2009-06-17 16:12:16 -06:00
|
|
|
// Save return value in m->cret
|
2010-08-04 18:50:22 -06:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ m(CX), BX
|
|
|
|
|
MOVQ AX, m_cret(BX)
|
2009-06-17 16:12:16 -06:00
|
|
|
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// Call oldstack on m->g0's stack.
|
|
|
|
|
MOVQ m_g0(BX), BP
|
|
|
|
|
MOVQ BP, g(CX)
|
|
|
|
|
MOVQ (g_sched+gobuf_sp)(BP), SP
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
CALL runtime·oldstack(SB)
|
2009-06-17 16:12:16 -06:00
|
|
|
MOVQ $0, 0x1004 // crash if oldstack returns
|
|
|
|
|
RET
|
|
|
|
|
|
2009-05-01 19:07:33 -06:00
|
|
|
// morestack trampolines
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·morestack00(SB),7,$0
|
2010-08-04 18:50:22 -06:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ m(CX), BX
|
2009-05-01 19:07:33 -06:00
|
|
|
MOVQ $0, AX
|
2011-01-14 12:05:20 -07:00
|
|
|
MOVQ AX, m_moreframesize(BX)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
MOVQ $runtime·morestack(SB), AX
|
2009-05-01 19:07:33 -06:00
|
|
|
JMP AX
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·morestack01(SB),7,$0
|
2010-08-04 18:50:22 -06:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ m(CX), BX
|
2009-05-01 19:07:33 -06:00
|
|
|
SHLQ $32, AX
|
2011-01-14 12:05:20 -07:00
|
|
|
MOVQ AX, m_moreframesize(BX)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
MOVQ $runtime·morestack(SB), AX
|
2009-05-01 19:07:33 -06:00
|
|
|
JMP AX
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·morestack10(SB),7,$0
|
2010-08-04 18:50:22 -06:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ m(CX), BX
|
2009-05-01 19:07:33 -06:00
|
|
|
MOVLQZX AX, AX
|
2011-01-14 12:05:20 -07:00
|
|
|
MOVQ AX, m_moreframesize(BX)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
MOVQ $runtime·morestack(SB), AX
|
2009-05-01 19:07:33 -06:00
|
|
|
JMP AX
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·morestack11(SB),7,$0
|
2010-08-04 18:50:22 -06:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ m(CX), BX
|
2011-01-14 12:05:20 -07:00
|
|
|
MOVQ AX, m_moreframesize(BX)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
MOVQ $runtime·morestack(SB), AX
|
2009-05-01 19:07:33 -06:00
|
|
|
JMP AX
|
|
|
|
|
|
2009-05-03 20:09:14 -06:00
|
|
|
// subcases of morestack01
|
|
|
|
|
// with const of 8,16,...48
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·morestack8(SB),7,$0
|
2009-05-03 20:09:14 -06:00
|
|
|
PUSHQ $1
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
MOVQ $morestack<>(SB), AX
|
2009-05-03 20:09:14 -06:00
|
|
|
JMP AX
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·morestack16(SB),7,$0
|
2009-05-03 20:09:14 -06:00
|
|
|
PUSHQ $2
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
MOVQ $morestack<>(SB), AX
|
2009-05-03 20:09:14 -06:00
|
|
|
JMP AX
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·morestack24(SB),7,$0
|
2009-05-03 20:09:14 -06:00
|
|
|
PUSHQ $3
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
MOVQ $morestack<>(SB), AX
|
2009-05-03 20:09:14 -06:00
|
|
|
JMP AX
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·morestack32(SB),7,$0
|
2009-05-03 20:09:14 -06:00
|
|
|
PUSHQ $4
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
MOVQ $morestack<>(SB), AX
|
2009-05-03 20:09:14 -06:00
|
|
|
JMP AX
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·morestack40(SB),7,$0
|
2009-05-03 20:09:14 -06:00
|
|
|
PUSHQ $5
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
MOVQ $morestack<>(SB), AX
|
2009-05-03 20:09:14 -06:00
|
|
|
JMP AX
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·morestack48(SB),7,$0
|
2009-05-03 20:09:14 -06:00
|
|
|
PUSHQ $6
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
MOVQ $morestack<>(SB), AX
|
2009-05-03 20:09:14 -06:00
|
|
|
JMP AX
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT morestack<>(SB),7,$0
|
2010-08-04 18:50:22 -06:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ m(CX), BX
|
2009-06-17 16:12:16 -06:00
|
|
|
POPQ AX
|
|
|
|
|
SHLQ $35, AX
|
2011-01-14 12:05:20 -07:00
|
|
|
MOVQ AX, m_moreframesize(BX)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
MOVQ $runtime·morestack(SB), AX
|
2008-07-12 12:30:53 -06:00
|
|
|
JMP AX
|
|
|
|
|
|
2008-08-04 17:43:49 -06:00
|
|
|
// bool cas(int32 *val, int32 old, int32 new)
|
|
|
|
|
// Atomically:
|
|
|
|
|
// if(*val == old){
|
|
|
|
|
// *val = new;
|
|
|
|
|
// return 1;
|
2009-01-27 13:03:53 -07:00
|
|
|
// } else
|
2008-08-04 17:43:49 -06:00
|
|
|
// return 0;
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·cas(SB), 7, $0
|
2008-08-04 17:43:49 -06:00
|
|
|
MOVQ 8(SP), BX
|
|
|
|
|
MOVL 16(SP), AX
|
|
|
|
|
MOVL 20(SP), CX
|
|
|
|
|
LOCK
|
|
|
|
|
CMPXCHGL CX, 0(BX)
|
|
|
|
|
JZ 3(PC)
|
|
|
|
|
MOVL $0, AX
|
|
|
|
|
RET
|
|
|
|
|
MOVL $1, AX
|
|
|
|
|
RET
|
2009-01-27 13:03:53 -07:00
|
|
|
|
2012-04-05 08:47:43 -06:00
|
|
|
// bool runtime·cas64(uint64 *val, uint64 *old, uint64 new)
|
|
|
|
|
// Atomically:
|
|
|
|
|
// if(*val == *old){
|
|
|
|
|
// *val = new;
|
|
|
|
|
// return 1;
|
|
|
|
|
// } else {
|
|
|
|
|
// *old = *val
|
|
|
|
|
// return 0;
|
|
|
|
|
// }
|
|
|
|
|
TEXT runtime·cas64(SB), 7, $0
|
|
|
|
|
MOVQ 8(SP), BX
|
|
|
|
|
MOVQ 16(SP), BP
|
|
|
|
|
MOVQ 0(BP), AX
|
|
|
|
|
MOVQ 24(SP), CX
|
|
|
|
|
LOCK
|
|
|
|
|
CMPXCHGQ CX, 0(BX)
|
|
|
|
|
JNZ cas64_fail
|
|
|
|
|
MOVL $1, AX
|
|
|
|
|
RET
|
|
|
|
|
cas64_fail:
|
|
|
|
|
MOVQ AX, 0(BP)
|
|
|
|
|
MOVL $0, AX
|
|
|
|
|
RET
|
|
|
|
|
|
2011-02-16 11:21:13 -07:00
|
|
|
// bool casp(void **val, void *old, void *new)
|
|
|
|
|
// Atomically:
|
|
|
|
|
// if(*val == old){
|
|
|
|
|
// *val = new;
|
|
|
|
|
// return 1;
|
|
|
|
|
// } else
|
|
|
|
|
// return 0;
|
|
|
|
|
TEXT runtime·casp(SB), 7, $0
|
|
|
|
|
MOVQ 8(SP), BX
|
|
|
|
|
MOVQ 16(SP), AX
|
|
|
|
|
MOVQ 24(SP), CX
|
|
|
|
|
LOCK
|
|
|
|
|
CMPXCHGQ CX, 0(BX)
|
|
|
|
|
JZ 3(PC)
|
|
|
|
|
MOVL $0, AX
|
|
|
|
|
RET
|
|
|
|
|
MOVL $1, AX
|
|
|
|
|
RET
|
|
|
|
|
|
2011-07-15 09:27:16 -06:00
|
|
|
// uint32 xadd(uint32 volatile *val, int32 delta)
|
|
|
|
|
// Atomically:
|
|
|
|
|
// *val += delta;
|
|
|
|
|
// return *val;
|
|
|
|
|
TEXT runtime·xadd(SB), 7, $0
|
|
|
|
|
MOVQ 8(SP), BX
|
|
|
|
|
MOVL 16(SP), AX
|
|
|
|
|
MOVL AX, CX
|
|
|
|
|
LOCK
|
|
|
|
|
XADDL AX, 0(BX)
|
|
|
|
|
ADDL CX, AX
|
|
|
|
|
RET
|
|
|
|
|
|
2012-04-05 08:47:43 -06:00
|
|
|
TEXT runtime·xadd64(SB), 7, $0
|
|
|
|
|
MOVQ 8(SP), BX
|
|
|
|
|
MOVQ 16(SP), AX
|
|
|
|
|
MOVQ AX, CX
|
|
|
|
|
LOCK
|
|
|
|
|
XADDQ AX, 0(BX)
|
|
|
|
|
ADDQ CX, AX
|
|
|
|
|
RET
|
|
|
|
|
|
runtime: improve Linux mutex
The implementation is hybrid active/passive spin/blocking mutex.
The design minimizes amount of context switches and futex calls.
The idea is that all critical sections in runtime are intentially
small, so pure blocking mutex behaves badly causing
a lot of context switches, thread parking/unparking and kernel calls.
Note that some synthetic benchmarks become somewhat slower,
that's due to increased contention on other data structures,
it should not affect programs that do any real work.
On 2 x Intel E5620, 8 HT cores, 2.4GHz
benchmark old ns/op new ns/op delta
BenchmarkSelectContended 521.00 503.00 -3.45%
BenchmarkSelectContended-2 661.00 320.00 -51.59%
BenchmarkSelectContended-4 1139.00 629.00 -44.78%
BenchmarkSelectContended-8 2870.00 878.00 -69.41%
BenchmarkSelectContended-16 5276.00 818.00 -84.50%
BenchmarkChanContended 112.00 103.00 -8.04%
BenchmarkChanContended-2 631.00 174.00 -72.42%
BenchmarkChanContended-4 682.00 272.00 -60.12%
BenchmarkChanContended-8 1601.00 520.00 -67.52%
BenchmarkChanContended-16 3100.00 372.00 -88.00%
BenchmarkChanSync 253.00 239.00 -5.53%
BenchmarkChanSync-2 5030.00 4648.00 -7.59%
BenchmarkChanSync-4 4826.00 4694.00 -2.74%
BenchmarkChanSync-8 4778.00 4713.00 -1.36%
BenchmarkChanSync-16 5289.00 4710.00 -10.95%
BenchmarkChanProdCons0 273.00 254.00 -6.96%
BenchmarkChanProdCons0-2 599.00 400.00 -33.22%
BenchmarkChanProdCons0-4 1168.00 659.00 -43.58%
BenchmarkChanProdCons0-8 2831.00 1057.00 -62.66%
BenchmarkChanProdCons0-16 4197.00 1037.00 -75.29%
BenchmarkChanProdCons10 150.00 140.00 -6.67%
BenchmarkChanProdCons10-2 607.00 268.00 -55.85%
BenchmarkChanProdCons10-4 1137.00 404.00 -64.47%
BenchmarkChanProdCons10-8 2115.00 828.00 -60.85%
BenchmarkChanProdCons10-16 4283.00 855.00 -80.04%
BenchmarkChanProdCons100 117.00 110.00 -5.98%
BenchmarkChanProdCons100-2 558.00 218.00 -60.93%
BenchmarkChanProdCons100-4 722.00 287.00 -60.25%
BenchmarkChanProdCons100-8 1840.00 431.00 -76.58%
BenchmarkChanProdCons100-16 3394.00 448.00 -86.80%
BenchmarkChanProdConsWork0 2014.00 1996.00 -0.89%
BenchmarkChanProdConsWork0-2 1207.00 1127.00 -6.63%
BenchmarkChanProdConsWork0-4 1913.00 611.00 -68.06%
BenchmarkChanProdConsWork0-8 3016.00 949.00 -68.53%
BenchmarkChanProdConsWork0-16 4320.00 1154.00 -73.29%
BenchmarkChanProdConsWork10 1906.00 1897.00 -0.47%
BenchmarkChanProdConsWork10-2 1123.00 1033.00 -8.01%
BenchmarkChanProdConsWork10-4 1076.00 571.00 -46.93%
BenchmarkChanProdConsWork10-8 2748.00 1096.00 -60.12%
BenchmarkChanProdConsWork10-16 4600.00 1105.00 -75.98%
BenchmarkChanProdConsWork100 1884.00 1852.00 -1.70%
BenchmarkChanProdConsWork100-2 1235.00 1146.00 -7.21%
BenchmarkChanProdConsWork100-4 1217.00 619.00 -49.14%
BenchmarkChanProdConsWork100-8 1534.00 509.00 -66.82%
BenchmarkChanProdConsWork100-16 4126.00 918.00 -77.75%
BenchmarkSyscall 34.40 33.30 -3.20%
BenchmarkSyscall-2 160.00 121.00 -24.38%
BenchmarkSyscall-4 131.00 136.00 +3.82%
BenchmarkSyscall-8 139.00 131.00 -5.76%
BenchmarkSyscall-16 161.00 168.00 +4.35%
BenchmarkSyscallWork 950.00 950.00 +0.00%
BenchmarkSyscallWork-2 481.00 480.00 -0.21%
BenchmarkSyscallWork-4 268.00 270.00 +0.75%
BenchmarkSyscallWork-8 156.00 169.00 +8.33%
BenchmarkSyscallWork-16 188.00 184.00 -2.13%
BenchmarkSemaSyntNonblock 36.40 35.60 -2.20%
BenchmarkSemaSyntNonblock-2 81.40 45.10 -44.59%
BenchmarkSemaSyntNonblock-4 126.00 108.00 -14.29%
BenchmarkSemaSyntNonblock-8 112.00 112.00 +0.00%
BenchmarkSemaSyntNonblock-16 110.00 112.00 +1.82%
BenchmarkSemaSyntBlock 35.30 35.30 +0.00%
BenchmarkSemaSyntBlock-2 118.00 124.00 +5.08%
BenchmarkSemaSyntBlock-4 105.00 108.00 +2.86%
BenchmarkSemaSyntBlock-8 101.00 111.00 +9.90%
BenchmarkSemaSyntBlock-16 112.00 118.00 +5.36%
BenchmarkSemaWorkNonblock 810.00 811.00 +0.12%
BenchmarkSemaWorkNonblock-2 476.00 414.00 -13.03%
BenchmarkSemaWorkNonblock-4 238.00 228.00 -4.20%
BenchmarkSemaWorkNonblock-8 140.00 126.00 -10.00%
BenchmarkSemaWorkNonblock-16 117.00 116.00 -0.85%
BenchmarkSemaWorkBlock 810.00 811.00 +0.12%
BenchmarkSemaWorkBlock-2 454.00 466.00 +2.64%
BenchmarkSemaWorkBlock-4 243.00 241.00 -0.82%
BenchmarkSemaWorkBlock-8 145.00 137.00 -5.52%
BenchmarkSemaWorkBlock-16 132.00 123.00 -6.82%
BenchmarkContendedSemaphore 123.00 102.00 -17.07%
BenchmarkContendedSemaphore-2 34.80 34.90 +0.29%
BenchmarkContendedSemaphore-4 34.70 34.80 +0.29%
BenchmarkContendedSemaphore-8 34.70 34.70 +0.00%
BenchmarkContendedSemaphore-16 34.80 34.70 -0.29%
BenchmarkMutex 26.80 26.00 -2.99%
BenchmarkMutex-2 108.00 45.20 -58.15%
BenchmarkMutex-4 103.00 127.00 +23.30%
BenchmarkMutex-8 109.00 147.00 +34.86%
BenchmarkMutex-16 102.00 152.00 +49.02%
BenchmarkMutexSlack 27.00 26.90 -0.37%
BenchmarkMutexSlack-2 149.00 165.00 +10.74%
BenchmarkMutexSlack-4 121.00 209.00 +72.73%
BenchmarkMutexSlack-8 101.00 158.00 +56.44%
BenchmarkMutexSlack-16 97.00 129.00 +32.99%
BenchmarkMutexWork 792.00 794.00 +0.25%
BenchmarkMutexWork-2 407.00 409.00 +0.49%
BenchmarkMutexWork-4 220.00 209.00 -5.00%
BenchmarkMutexWork-8 267.00 160.00 -40.07%
BenchmarkMutexWork-16 315.00 300.00 -4.76%
BenchmarkMutexWorkSlack 792.00 793.00 +0.13%
BenchmarkMutexWorkSlack-2 406.00 404.00 -0.49%
BenchmarkMutexWorkSlack-4 225.00 212.00 -5.78%
BenchmarkMutexWorkSlack-8 268.00 136.00 -49.25%
BenchmarkMutexWorkSlack-16 300.00 300.00 +0.00%
BenchmarkRWMutexWrite100 27.10 27.00 -0.37%
BenchmarkRWMutexWrite100-2 33.10 40.80 +23.26%
BenchmarkRWMutexWrite100-4 113.00 88.10 -22.04%
BenchmarkRWMutexWrite100-8 119.00 95.30 -19.92%
BenchmarkRWMutexWrite100-16 148.00 109.00 -26.35%
BenchmarkRWMutexWrite10 29.60 29.40 -0.68%
BenchmarkRWMutexWrite10-2 111.00 61.40 -44.68%
BenchmarkRWMutexWrite10-4 270.00 208.00 -22.96%
BenchmarkRWMutexWrite10-8 204.00 185.00 -9.31%
BenchmarkRWMutexWrite10-16 261.00 190.00 -27.20%
BenchmarkRWMutexWorkWrite100 1040.00 1036.00 -0.38%
BenchmarkRWMutexWorkWrite100-2 593.00 580.00 -2.19%
BenchmarkRWMutexWorkWrite100-4 470.00 365.00 -22.34%
BenchmarkRWMutexWorkWrite100-8 468.00 289.00 -38.25%
BenchmarkRWMutexWorkWrite100-16 604.00 374.00 -38.08%
BenchmarkRWMutexWorkWrite10 951.00 951.00 +0.00%
BenchmarkRWMutexWorkWrite10-2 1001.00 928.00 -7.29%
BenchmarkRWMutexWorkWrite10-4 1555.00 1006.00 -35.31%
BenchmarkRWMutexWorkWrite10-8 2085.00 1171.00 -43.84%
BenchmarkRWMutexWorkWrite10-16 2082.00 1614.00 -22.48%
R=rsc, iant, msolo, fw, iant
CC=golang-dev
https://golang.org/cl/4711045
2011-07-29 10:44:06 -06:00
|
|
|
TEXT runtime·xchg(SB), 7, $0
|
|
|
|
|
MOVQ 8(SP), BX
|
|
|
|
|
MOVL 16(SP), AX
|
|
|
|
|
XCHGL AX, 0(BX)
|
|
|
|
|
RET
|
|
|
|
|
|
2013-03-05 00:46:52 -07:00
|
|
|
TEXT runtime·xchg64(SB), 7, $0
|
|
|
|
|
MOVQ 8(SP), BX
|
|
|
|
|
MOVQ 16(SP), AX
|
|
|
|
|
XCHGQ AX, 0(BX)
|
|
|
|
|
RET
|
|
|
|
|
|
runtime: improve Linux mutex
The implementation is hybrid active/passive spin/blocking mutex.
The design minimizes amount of context switches and futex calls.
The idea is that all critical sections in runtime are intentially
small, so pure blocking mutex behaves badly causing
a lot of context switches, thread parking/unparking and kernel calls.
Note that some synthetic benchmarks become somewhat slower,
that's due to increased contention on other data structures,
it should not affect programs that do any real work.
On 2 x Intel E5620, 8 HT cores, 2.4GHz
benchmark old ns/op new ns/op delta
BenchmarkSelectContended 521.00 503.00 -3.45%
BenchmarkSelectContended-2 661.00 320.00 -51.59%
BenchmarkSelectContended-4 1139.00 629.00 -44.78%
BenchmarkSelectContended-8 2870.00 878.00 -69.41%
BenchmarkSelectContended-16 5276.00 818.00 -84.50%
BenchmarkChanContended 112.00 103.00 -8.04%
BenchmarkChanContended-2 631.00 174.00 -72.42%
BenchmarkChanContended-4 682.00 272.00 -60.12%
BenchmarkChanContended-8 1601.00 520.00 -67.52%
BenchmarkChanContended-16 3100.00 372.00 -88.00%
BenchmarkChanSync 253.00 239.00 -5.53%
BenchmarkChanSync-2 5030.00 4648.00 -7.59%
BenchmarkChanSync-4 4826.00 4694.00 -2.74%
BenchmarkChanSync-8 4778.00 4713.00 -1.36%
BenchmarkChanSync-16 5289.00 4710.00 -10.95%
BenchmarkChanProdCons0 273.00 254.00 -6.96%
BenchmarkChanProdCons0-2 599.00 400.00 -33.22%
BenchmarkChanProdCons0-4 1168.00 659.00 -43.58%
BenchmarkChanProdCons0-8 2831.00 1057.00 -62.66%
BenchmarkChanProdCons0-16 4197.00 1037.00 -75.29%
BenchmarkChanProdCons10 150.00 140.00 -6.67%
BenchmarkChanProdCons10-2 607.00 268.00 -55.85%
BenchmarkChanProdCons10-4 1137.00 404.00 -64.47%
BenchmarkChanProdCons10-8 2115.00 828.00 -60.85%
BenchmarkChanProdCons10-16 4283.00 855.00 -80.04%
BenchmarkChanProdCons100 117.00 110.00 -5.98%
BenchmarkChanProdCons100-2 558.00 218.00 -60.93%
BenchmarkChanProdCons100-4 722.00 287.00 -60.25%
BenchmarkChanProdCons100-8 1840.00 431.00 -76.58%
BenchmarkChanProdCons100-16 3394.00 448.00 -86.80%
BenchmarkChanProdConsWork0 2014.00 1996.00 -0.89%
BenchmarkChanProdConsWork0-2 1207.00 1127.00 -6.63%
BenchmarkChanProdConsWork0-4 1913.00 611.00 -68.06%
BenchmarkChanProdConsWork0-8 3016.00 949.00 -68.53%
BenchmarkChanProdConsWork0-16 4320.00 1154.00 -73.29%
BenchmarkChanProdConsWork10 1906.00 1897.00 -0.47%
BenchmarkChanProdConsWork10-2 1123.00 1033.00 -8.01%
BenchmarkChanProdConsWork10-4 1076.00 571.00 -46.93%
BenchmarkChanProdConsWork10-8 2748.00 1096.00 -60.12%
BenchmarkChanProdConsWork10-16 4600.00 1105.00 -75.98%
BenchmarkChanProdConsWork100 1884.00 1852.00 -1.70%
BenchmarkChanProdConsWork100-2 1235.00 1146.00 -7.21%
BenchmarkChanProdConsWork100-4 1217.00 619.00 -49.14%
BenchmarkChanProdConsWork100-8 1534.00 509.00 -66.82%
BenchmarkChanProdConsWork100-16 4126.00 918.00 -77.75%
BenchmarkSyscall 34.40 33.30 -3.20%
BenchmarkSyscall-2 160.00 121.00 -24.38%
BenchmarkSyscall-4 131.00 136.00 +3.82%
BenchmarkSyscall-8 139.00 131.00 -5.76%
BenchmarkSyscall-16 161.00 168.00 +4.35%
BenchmarkSyscallWork 950.00 950.00 +0.00%
BenchmarkSyscallWork-2 481.00 480.00 -0.21%
BenchmarkSyscallWork-4 268.00 270.00 +0.75%
BenchmarkSyscallWork-8 156.00 169.00 +8.33%
BenchmarkSyscallWork-16 188.00 184.00 -2.13%
BenchmarkSemaSyntNonblock 36.40 35.60 -2.20%
BenchmarkSemaSyntNonblock-2 81.40 45.10 -44.59%
BenchmarkSemaSyntNonblock-4 126.00 108.00 -14.29%
BenchmarkSemaSyntNonblock-8 112.00 112.00 +0.00%
BenchmarkSemaSyntNonblock-16 110.00 112.00 +1.82%
BenchmarkSemaSyntBlock 35.30 35.30 +0.00%
BenchmarkSemaSyntBlock-2 118.00 124.00 +5.08%
BenchmarkSemaSyntBlock-4 105.00 108.00 +2.86%
BenchmarkSemaSyntBlock-8 101.00 111.00 +9.90%
BenchmarkSemaSyntBlock-16 112.00 118.00 +5.36%
BenchmarkSemaWorkNonblock 810.00 811.00 +0.12%
BenchmarkSemaWorkNonblock-2 476.00 414.00 -13.03%
BenchmarkSemaWorkNonblock-4 238.00 228.00 -4.20%
BenchmarkSemaWorkNonblock-8 140.00 126.00 -10.00%
BenchmarkSemaWorkNonblock-16 117.00 116.00 -0.85%
BenchmarkSemaWorkBlock 810.00 811.00 +0.12%
BenchmarkSemaWorkBlock-2 454.00 466.00 +2.64%
BenchmarkSemaWorkBlock-4 243.00 241.00 -0.82%
BenchmarkSemaWorkBlock-8 145.00 137.00 -5.52%
BenchmarkSemaWorkBlock-16 132.00 123.00 -6.82%
BenchmarkContendedSemaphore 123.00 102.00 -17.07%
BenchmarkContendedSemaphore-2 34.80 34.90 +0.29%
BenchmarkContendedSemaphore-4 34.70 34.80 +0.29%
BenchmarkContendedSemaphore-8 34.70 34.70 +0.00%
BenchmarkContendedSemaphore-16 34.80 34.70 -0.29%
BenchmarkMutex 26.80 26.00 -2.99%
BenchmarkMutex-2 108.00 45.20 -58.15%
BenchmarkMutex-4 103.00 127.00 +23.30%
BenchmarkMutex-8 109.00 147.00 +34.86%
BenchmarkMutex-16 102.00 152.00 +49.02%
BenchmarkMutexSlack 27.00 26.90 -0.37%
BenchmarkMutexSlack-2 149.00 165.00 +10.74%
BenchmarkMutexSlack-4 121.00 209.00 +72.73%
BenchmarkMutexSlack-8 101.00 158.00 +56.44%
BenchmarkMutexSlack-16 97.00 129.00 +32.99%
BenchmarkMutexWork 792.00 794.00 +0.25%
BenchmarkMutexWork-2 407.00 409.00 +0.49%
BenchmarkMutexWork-4 220.00 209.00 -5.00%
BenchmarkMutexWork-8 267.00 160.00 -40.07%
BenchmarkMutexWork-16 315.00 300.00 -4.76%
BenchmarkMutexWorkSlack 792.00 793.00 +0.13%
BenchmarkMutexWorkSlack-2 406.00 404.00 -0.49%
BenchmarkMutexWorkSlack-4 225.00 212.00 -5.78%
BenchmarkMutexWorkSlack-8 268.00 136.00 -49.25%
BenchmarkMutexWorkSlack-16 300.00 300.00 +0.00%
BenchmarkRWMutexWrite100 27.10 27.00 -0.37%
BenchmarkRWMutexWrite100-2 33.10 40.80 +23.26%
BenchmarkRWMutexWrite100-4 113.00 88.10 -22.04%
BenchmarkRWMutexWrite100-8 119.00 95.30 -19.92%
BenchmarkRWMutexWrite100-16 148.00 109.00 -26.35%
BenchmarkRWMutexWrite10 29.60 29.40 -0.68%
BenchmarkRWMutexWrite10-2 111.00 61.40 -44.68%
BenchmarkRWMutexWrite10-4 270.00 208.00 -22.96%
BenchmarkRWMutexWrite10-8 204.00 185.00 -9.31%
BenchmarkRWMutexWrite10-16 261.00 190.00 -27.20%
BenchmarkRWMutexWorkWrite100 1040.00 1036.00 -0.38%
BenchmarkRWMutexWorkWrite100-2 593.00 580.00 -2.19%
BenchmarkRWMutexWorkWrite100-4 470.00 365.00 -22.34%
BenchmarkRWMutexWorkWrite100-8 468.00 289.00 -38.25%
BenchmarkRWMutexWorkWrite100-16 604.00 374.00 -38.08%
BenchmarkRWMutexWorkWrite10 951.00 951.00 +0.00%
BenchmarkRWMutexWorkWrite10-2 1001.00 928.00 -7.29%
BenchmarkRWMutexWorkWrite10-4 1555.00 1006.00 -35.31%
BenchmarkRWMutexWorkWrite10-8 2085.00 1171.00 -43.84%
BenchmarkRWMutexWorkWrite10-16 2082.00 1614.00 -22.48%
R=rsc, iant, msolo, fw, iant
CC=golang-dev
https://golang.org/cl/4711045
2011-07-29 10:44:06 -06:00
|
|
|
TEXT runtime·procyield(SB),7,$0
|
|
|
|
|
MOVL 8(SP), AX
|
|
|
|
|
again:
|
|
|
|
|
PAUSE
|
|
|
|
|
SUBL $1, AX
|
|
|
|
|
JNZ again
|
|
|
|
|
RET
|
|
|
|
|
|
2011-07-13 12:22:41 -06:00
|
|
|
TEXT runtime·atomicstorep(SB), 7, $0
|
|
|
|
|
MOVQ 8(SP), BX
|
|
|
|
|
MOVQ 16(SP), AX
|
|
|
|
|
XCHGQ AX, 0(BX)
|
|
|
|
|
RET
|
|
|
|
|
|
2011-07-29 11:47:24 -06:00
|
|
|
TEXT runtime·atomicstore(SB), 7, $0
|
|
|
|
|
MOVQ 8(SP), BX
|
|
|
|
|
MOVL 16(SP), AX
|
|
|
|
|
XCHGL AX, 0(BX)
|
|
|
|
|
RET
|
|
|
|
|
|
2012-04-05 08:47:43 -06:00
|
|
|
TEXT runtime·atomicstore64(SB), 7, $0
|
|
|
|
|
MOVQ 8(SP), BX
|
|
|
|
|
MOVQ 16(SP), AX
|
|
|
|
|
XCHGQ AX, 0(BX)
|
|
|
|
|
RET
|
|
|
|
|
|
2009-06-03 00:02:12 -06:00
|
|
|
// void jmpdefer(fn, sp);
|
|
|
|
|
// called from deferreturn.
|
2009-01-27 13:03:53 -07:00
|
|
|
// 1. pop the caller
|
|
|
|
|
// 2. sub 5 bytes from the callers return
|
|
|
|
|
// 3. jmp to the argument
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·jmpdefer(SB), 7, $0
|
2013-02-22 08:47:54 -07:00
|
|
|
MOVQ 8(SP), DX // fn
|
2009-06-03 00:02:12 -06:00
|
|
|
MOVQ 16(SP), BX // caller sp
|
|
|
|
|
LEAQ -8(BX), SP // caller sp after CALL
|
|
|
|
|
SUBQ $5, (SP) // return to CALL again
|
2013-02-22 08:47:54 -07:00
|
|
|
MOVQ 0(DX), BX
|
2013-02-21 15:01:13 -07:00
|
|
|
JMP BX // but first run the deferred function
|
2009-10-03 11:37:12 -06:00
|
|
|
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// Dummy function to use in saved gobuf.PC,
|
|
|
|
|
// to match SP pointing at a return address.
|
|
|
|
|
// The gobuf.PC is unused by the contortions here
|
|
|
|
|
// but setting it to return will make the traceback code work.
|
|
|
|
|
TEXT return<>(SB),7,$0
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
// asmcgocall(void(*fn)(void*), void *arg)
|
2009-10-12 11:26:38 -06:00
|
|
|
// Call fn(arg) on the scheduler stack,
|
|
|
|
|
// aligned appropriately for the gcc ABI.
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// See cgocall.c for more details.
|
|
|
|
|
TEXT runtime·asmcgocall(SB),7,$0
|
|
|
|
|
MOVQ fn+0(FP), AX
|
|
|
|
|
MOVQ arg+8(FP), BX
|
|
|
|
|
MOVQ SP, DX
|
2009-10-12 11:26:38 -06:00
|
|
|
|
|
|
|
|
// Figure out if we need to switch to m->g0 stack.
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// We get called to create new OS threads too, and those
|
|
|
|
|
// come in on the m->g0 stack already.
|
|
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ m(CX), BP
|
|
|
|
|
MOVQ m_g0(BP), SI
|
|
|
|
|
MOVQ g(CX), DI
|
|
|
|
|
CMPQ SI, DI
|
|
|
|
|
JEQ 6(PC)
|
|
|
|
|
MOVQ SP, (g_sched+gobuf_sp)(DI)
|
|
|
|
|
MOVQ $return<>(SB), (g_sched+gobuf_pc)(DI)
|
|
|
|
|
MOVQ DI, (g_sched+gobuf_g)(DI)
|
|
|
|
|
MOVQ SI, g(CX)
|
|
|
|
|
MOVQ (g_sched+gobuf_sp)(SI), SP
|
2009-10-12 11:26:38 -06:00
|
|
|
|
|
|
|
|
// Now on a scheduling stack (a pthread-created stack).
|
2012-09-02 20:12:51 -06:00
|
|
|
// Make sure we have enough room for 4 stack-backed fast-call
|
|
|
|
|
// registers as per windows amd64 calling convention.
|
|
|
|
|
SUBQ $64, SP
|
2009-10-03 11:37:12 -06:00
|
|
|
ANDQ $~15, SP // alignment for gcc ABI
|
2012-09-02 20:12:51 -06:00
|
|
|
MOVQ DI, 48(SP) // save g
|
|
|
|
|
MOVQ DX, 40(SP) // save SP
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ BX, DI // DI = first argument in AMD64 ABI
|
2011-07-19 08:47:33 -06:00
|
|
|
MOVQ BX, CX // CX = first argument in Win64
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
CALL AX
|
2009-10-12 11:26:38 -06:00
|
|
|
|
2010-08-04 18:50:22 -06:00
|
|
|
// Restore registers, g, stack pointer.
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
get_tls(CX)
|
2012-09-02 20:12:51 -06:00
|
|
|
MOVQ 48(SP), DI
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ DI, g(CX)
|
2012-09-02 20:12:51 -06:00
|
|
|
MOVQ 40(SP), SP
|
2009-10-03 11:37:12 -06:00
|
|
|
RET
|
|
|
|
|
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// cgocallback(void (*fn)(void*), void *frame, uintptr framesize)
|
2013-02-22 14:08:56 -07:00
|
|
|
// Turn the fn into a Go func (by taking its address) and call
|
|
|
|
|
// cgocallback_gofunc.
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
TEXT runtime·cgocallback(SB),7,$24
|
2013-02-22 14:08:56 -07:00
|
|
|
LEAQ fn+0(FP), AX
|
|
|
|
|
MOVQ AX, 0(SP)
|
|
|
|
|
MOVQ frame+8(FP), AX
|
|
|
|
|
MOVQ AX, 8(SP)
|
|
|
|
|
MOVQ framesize+16(FP), AX
|
|
|
|
|
MOVQ AX, 16(SP)
|
|
|
|
|
MOVQ $runtime·cgocallback_gofunc(SB), AX
|
|
|
|
|
CALL AX
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize)
|
|
|
|
|
// See cgocall.c for more details.
|
|
|
|
|
TEXT runtime·cgocallback_gofunc(SB),7,$24
|
2013-02-20 15:48:23 -07:00
|
|
|
// If m is nil, Go did not create the current thread.
|
|
|
|
|
// Call needm to obtain one for temporary use.
|
|
|
|
|
// In this case, we're running on the thread stack, so there's
|
|
|
|
|
// lots of space, but the linker doesn't know. Hide the call from
|
|
|
|
|
// the linker analysis by using an indirect call through AX.
|
2010-08-04 18:50:22 -06:00
|
|
|
get_tls(CX)
|
2013-02-20 15:48:23 -07:00
|
|
|
#ifdef GOOS_windows
|
|
|
|
|
CMPQ CX, $0
|
|
|
|
|
JNE 3(PC)
|
|
|
|
|
PUSHQ $0
|
|
|
|
|
JMP needm
|
|
|
|
|
#endif
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ m(CX), BP
|
2013-02-20 15:48:23 -07:00
|
|
|
PUSHQ BP
|
2012-03-08 10:12:40 -07:00
|
|
|
CMPQ BP, $0
|
2013-02-20 15:48:23 -07:00
|
|
|
JNE havem
|
|
|
|
|
needm:
|
|
|
|
|
MOVQ $runtime·needm(SB), AX
|
|
|
|
|
CALL AX
|
|
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ m(CX), BP
|
2012-03-08 10:12:40 -07:00
|
|
|
|
2013-02-20 15:48:23 -07:00
|
|
|
havem:
|
|
|
|
|
// Now there's a valid m, and we're running on its m->g0.
|
|
|
|
|
// Save current m->g0->sched.sp on stack and then set it to SP.
|
|
|
|
|
// Save current sp in m->g0->sched.sp in preparation for
|
|
|
|
|
// switch back to m->curg stack.
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ m_g0(BP), SI
|
|
|
|
|
PUSHQ (g_sched+gobuf_sp)(SI)
|
|
|
|
|
MOVQ SP, (g_sched+gobuf_sp)(SI)
|
|
|
|
|
|
2011-08-18 10:17:09 -06:00
|
|
|
// Switch to m->curg stack and call runtime.cgocallbackg
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// with the three arguments. Because we are taking over
|
|
|
|
|
// the execution of m->curg but *not* resuming what had
|
|
|
|
|
// been running, we need to save that information (m->curg->gobuf)
|
|
|
|
|
// so that we can restore it when we're done.
|
|
|
|
|
// We can restore m->curg->gobuf.sp easily, because calling
|
2011-08-18 10:17:09 -06:00
|
|
|
// runtime.cgocallbackg leaves SP unchanged upon return.
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// To save m->curg->gobuf.pc, we push it onto the stack.
|
|
|
|
|
// This has the added benefit that it looks to the traceback
|
2011-08-18 10:17:09 -06:00
|
|
|
// routine like cgocallbackg is going to return to that
|
|
|
|
|
// PC (because we defined cgocallbackg to have
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// a frame size of 24, the same amount that we use below),
|
|
|
|
|
// so that the traceback will seamlessly trace back into
|
|
|
|
|
// the earlier calls.
|
2013-02-20 15:48:23 -07:00
|
|
|
MOVQ fn+0(FP), AX
|
|
|
|
|
MOVQ frame+8(FP), BX
|
|
|
|
|
MOVQ framesize+16(FP), DX
|
|
|
|
|
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ m_curg(BP), SI
|
|
|
|
|
MOVQ SI, g(CX)
|
|
|
|
|
MOVQ (g_sched+gobuf_sp)(SI), DI // prepare stack as DI
|
|
|
|
|
|
|
|
|
|
// Push gobuf.pc
|
|
|
|
|
MOVQ (g_sched+gobuf_pc)(SI), BP
|
|
|
|
|
SUBQ $8, DI
|
|
|
|
|
MOVQ BP, 0(DI)
|
|
|
|
|
|
|
|
|
|
// Push arguments to cgocallbackg.
|
|
|
|
|
// Frame size here must match the frame size above
|
|
|
|
|
// to trick traceback routines into doing the right thing.
|
|
|
|
|
SUBQ $24, DI
|
|
|
|
|
MOVQ AX, 0(DI)
|
|
|
|
|
MOVQ BX, 8(DI)
|
|
|
|
|
MOVQ DX, 16(DI)
|
|
|
|
|
|
|
|
|
|
// Switch stack and make the call.
|
|
|
|
|
MOVQ DI, SP
|
|
|
|
|
CALL runtime·cgocallbackg(SB)
|
|
|
|
|
|
|
|
|
|
// Restore g->gobuf (== m->curg->gobuf) from saved values.
|
|
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ g(CX), SI
|
|
|
|
|
MOVQ 24(SP), BP
|
|
|
|
|
MOVQ BP, (g_sched+gobuf_pc)(SI)
|
|
|
|
|
LEAQ (24+8)(SP), DI
|
|
|
|
|
MOVQ DI, (g_sched+gobuf_sp)(SI)
|
|
|
|
|
|
|
|
|
|
// Switch back to m->g0's stack and restore m->g0->sched.sp.
|
|
|
|
|
// (Unlike m->curg, the g0 goroutine never uses sched.pc,
|
|
|
|
|
// so we do not have to restore it.)
|
|
|
|
|
MOVQ m(CX), BP
|
|
|
|
|
MOVQ m_g0(BP), SI
|
|
|
|
|
MOVQ SI, g(CX)
|
|
|
|
|
MOVQ (g_sched+gobuf_sp)(SI), SP
|
|
|
|
|
POPQ (g_sched+gobuf_sp)(SI)
|
2013-02-20 15:48:23 -07:00
|
|
|
|
|
|
|
|
// If the m on entry was nil, we called needm above to borrow an m
|
|
|
|
|
// for the duration of the call. Since the call is over, return it with dropm.
|
|
|
|
|
POPQ BP
|
|
|
|
|
CMPQ BP, $0
|
|
|
|
|
JNE 3(PC)
|
|
|
|
|
MOVQ $runtime·dropm(SB), AX
|
|
|
|
|
CALL AX
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
|
|
|
|
|
// Done!
|
2010-04-09 14:30:11 -06:00
|
|
|
RET
|
|
|
|
|
|
2013-02-20 15:48:23 -07:00
|
|
|
// void setmg(M*, G*); set m and g. for use by needm.
|
|
|
|
|
TEXT runtime·setmg(SB), 7, $0
|
|
|
|
|
MOVQ mm+0(FP), AX
|
|
|
|
|
#ifdef GOOS_windows
|
|
|
|
|
CMPQ AX, $0
|
|
|
|
|
JNE settls
|
|
|
|
|
MOVQ $0, 0x28(GS)
|
|
|
|
|
RET
|
|
|
|
|
settls:
|
|
|
|
|
LEAQ m_tls(AX), AX
|
|
|
|
|
MOVQ AX, 0x28(GS)
|
|
|
|
|
#endif
|
|
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ mm+0(FP), AX
|
|
|
|
|
MOVQ AX, m(CX)
|
|
|
|
|
MOVQ gg+8(FP), BX
|
|
|
|
|
MOVQ BX, g(CX)
|
|
|
|
|
RET
|
|
|
|
|
|
2013-03-25 16:14:02 -06:00
|
|
|
// void setmg_gcc(M*, G*); set m and g called from gcc.
|
|
|
|
|
TEXT setmg_gcc<>(SB),7,$0
|
|
|
|
|
get_tls(AX)
|
|
|
|
|
MOVQ DI, m(AX)
|
|
|
|
|
MOVQ SI, g(AX)
|
|
|
|
|
RET
|
|
|
|
|
|
2009-12-08 19:19:30 -07:00
|
|
|
// check that SP is in range [g->stackbase, g->stackguard)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·stackcheck(SB), 7, $0
|
2010-08-04 18:50:22 -06:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ g(CX), AX
|
|
|
|
|
CMPQ g_stackbase(AX), SP
|
2010-03-30 11:53:16 -06:00
|
|
|
JHI 2(PC)
|
|
|
|
|
INT $3
|
2010-08-04 18:50:22 -06:00
|
|
|
CMPQ SP, g_stackguard(AX)
|
2010-03-30 11:53:16 -06:00
|
|
|
JHI 2(PC)
|
|
|
|
|
INT $3
|
|
|
|
|
RET
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·memclr(SB),7,$0
|
2010-04-05 13:51:09 -06:00
|
|
|
MOVQ 8(SP), DI // arg 1 addr
|
2011-01-25 14:35:36 -07:00
|
|
|
MOVQ 16(SP), CX // arg 2 count
|
2011-07-23 13:46:58 -06:00
|
|
|
MOVQ CX, BX
|
|
|
|
|
ANDQ $7, BX
|
2011-01-25 14:35:36 -07:00
|
|
|
SHRQ $3, CX
|
2010-04-05 13:51:09 -06:00
|
|
|
MOVQ $0, AX
|
|
|
|
|
CLD
|
|
|
|
|
REP
|
|
|
|
|
STOSQ
|
2011-07-23 13:46:58 -06:00
|
|
|
MOVQ BX, CX
|
|
|
|
|
REP
|
|
|
|
|
STOSB
|
2010-04-05 13:51:09 -06:00
|
|
|
RET
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·getcallerpc(SB),7,$0
|
2010-04-05 13:51:09 -06:00
|
|
|
MOVQ x+0(FP),AX // addr of first arg
|
|
|
|
|
MOVQ -8(AX),AX // get calling pc
|
|
|
|
|
RET
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·setcallerpc(SB),7,$0
|
2010-04-05 13:51:09 -06:00
|
|
|
MOVQ x+0(FP),AX // addr of first arg
|
|
|
|
|
MOVQ x+8(FP), BX
|
|
|
|
|
MOVQ BX, -8(AX) // set calling pc
|
|
|
|
|
RET
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
TEXT runtime·getcallersp(SB),7,$0
|
2010-04-05 13:51:09 -06:00
|
|
|
MOVQ sp+0(FP), AX
|
|
|
|
|
RET
|
|
|
|
|
|
2012-02-02 12:09:27 -07:00
|
|
|
// int64 runtime·cputicks(void)
|
|
|
|
|
TEXT runtime·cputicks(SB),7,$0
|
|
|
|
|
RDTSC
|
|
|
|
|
SHLQ $32, DX
|
|
|
|
|
ADDQ DX, AX
|
|
|
|
|
RET
|
|
|
|
|
|
2012-03-15 13:22:30 -06:00
|
|
|
TEXT runtime·stackguard(SB),7,$0
|
|
|
|
|
MOVQ SP, DX
|
|
|
|
|
MOVQ DX, sp+0(FP)
|
|
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ g(CX), BX
|
|
|
|
|
MOVQ g_stackguard(BX), DX
|
2013-03-22 10:57:55 -06:00
|
|
|
MOVQ DX, limit+8(FP)
|
2012-03-15 13:22:30 -06:00
|
|
|
RET
|
|
|
|
|
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
GLOBL runtime·tls0(SB), $64
|
2013-03-12 11:47:44 -06:00
|
|
|
|
|
|
|
|
// hash function using AES hardware instructions
|
|
|
|
|
TEXT runtime·aeshash(SB),7,$0
|
|
|
|
|
MOVQ 8(SP), DX // ptr to hash value
|
|
|
|
|
MOVQ 16(SP), CX // size
|
|
|
|
|
MOVQ 24(SP), AX // ptr to data
|
|
|
|
|
JMP runtime·aeshashbody(SB)
|
|
|
|
|
|
|
|
|
|
TEXT runtime·aeshashstr(SB),7,$0
|
|
|
|
|
MOVQ 8(SP), DX // ptr to hash value
|
|
|
|
|
MOVQ 24(SP), AX // ptr to string struct
|
|
|
|
|
MOVQ 8(AX), CX // length of string
|
|
|
|
|
MOVQ (AX), AX // string data
|
|
|
|
|
JMP runtime·aeshashbody(SB)
|
|
|
|
|
|
|
|
|
|
// AX: data
|
|
|
|
|
// CX: length
|
|
|
|
|
// DX: ptr to seed input / hash output
|
|
|
|
|
TEXT runtime·aeshashbody(SB),7,$0
|
|
|
|
|
MOVQ (DX), X0 // seed to low 64 bits of xmm0
|
|
|
|
|
PINSRQ $1, CX, X0 // size to high 64 bits of xmm0
|
2013-03-20 15:34:26 -06:00
|
|
|
MOVO runtime·aeskeysched+0(SB), X2
|
|
|
|
|
MOVO runtime·aeskeysched+16(SB), X3
|
2013-03-12 11:47:44 -06:00
|
|
|
aesloop:
|
|
|
|
|
CMPQ CX, $16
|
|
|
|
|
JB aesloopend
|
|
|
|
|
MOVOU (AX), X1
|
|
|
|
|
AESENC X2, X0
|
|
|
|
|
AESENC X1, X0
|
|
|
|
|
SUBQ $16, CX
|
|
|
|
|
ADDQ $16, AX
|
|
|
|
|
JMP aesloop
|
|
|
|
|
aesloopend:
|
|
|
|
|
TESTQ CX, CX
|
|
|
|
|
JE finalize // no partial block
|
|
|
|
|
|
|
|
|
|
TESTQ $16, AX
|
|
|
|
|
JNE highpartial
|
|
|
|
|
|
|
|
|
|
// address ends in 0xxxx. 16 bytes loaded
|
|
|
|
|
// at this address won't cross a page boundary, so
|
|
|
|
|
// we can load it directly.
|
|
|
|
|
MOVOU (AX), X1
|
|
|
|
|
ADDQ CX, CX
|
|
|
|
|
PAND masks(SB)(CX*8), X1
|
|
|
|
|
JMP partial
|
|
|
|
|
highpartial:
|
|
|
|
|
// address ends in 1xxxx. Might be up against
|
|
|
|
|
// a page boundary, so load ending at last byte.
|
|
|
|
|
// Then shift bytes down using pshufb.
|
|
|
|
|
MOVOU -16(AX)(CX*1), X1
|
|
|
|
|
ADDQ CX, CX
|
|
|
|
|
PSHUFB shifts(SB)(CX*8), X1
|
|
|
|
|
partial:
|
|
|
|
|
// incorporate partial block into hash
|
|
|
|
|
AESENC X3, X0
|
|
|
|
|
AESENC X1, X0
|
|
|
|
|
finalize:
|
|
|
|
|
// finalize hash
|
|
|
|
|
AESENC X2, X0
|
|
|
|
|
AESENC X3, X0
|
|
|
|
|
AESENC X2, X0
|
|
|
|
|
MOVQ X0, (DX)
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
TEXT runtime·aeshash32(SB),7,$0
|
|
|
|
|
MOVQ 8(SP), DX // ptr to hash value
|
|
|
|
|
MOVQ 24(SP), AX // ptr to data
|
|
|
|
|
MOVQ (DX), X0 // seed
|
|
|
|
|
PINSRD $2, (AX), X0 // data
|
2013-03-20 15:34:26 -06:00
|
|
|
AESENC runtime·aeskeysched+0(SB), X0
|
|
|
|
|
AESENC runtime·aeskeysched+16(SB), X0
|
|
|
|
|
AESENC runtime·aeskeysched+0(SB), X0
|
2013-03-12 11:47:44 -06:00
|
|
|
MOVQ X0, (DX)
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
TEXT runtime·aeshash64(SB),7,$0
|
|
|
|
|
MOVQ 8(SP), DX // ptr to hash value
|
|
|
|
|
MOVQ 24(SP), AX // ptr to data
|
|
|
|
|
MOVQ (DX), X0 // seed
|
|
|
|
|
PINSRQ $1, (AX), X0 // data
|
2013-03-20 15:34:26 -06:00
|
|
|
AESENC runtime·aeskeysched+0(SB), X0
|
|
|
|
|
AESENC runtime·aeskeysched+16(SB), X0
|
|
|
|
|
AESENC runtime·aeskeysched+0(SB), X0
|
2013-03-12 11:47:44 -06:00
|
|
|
MOVQ X0, (DX)
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
// simple mask to get rid of data in the high part of the register.
|
|
|
|
|
TEXT masks(SB),7,$0
|
|
|
|
|
QUAD $0x0000000000000000
|
|
|
|
|
QUAD $0x0000000000000000
|
|
|
|
|
QUAD $0x00000000000000ff
|
|
|
|
|
QUAD $0x0000000000000000
|
|
|
|
|
QUAD $0x000000000000ffff
|
|
|
|
|
QUAD $0x0000000000000000
|
|
|
|
|
QUAD $0x0000000000ffffff
|
|
|
|
|
QUAD $0x0000000000000000
|
|
|
|
|
QUAD $0x00000000ffffffff
|
|
|
|
|
QUAD $0x0000000000000000
|
|
|
|
|
QUAD $0x000000ffffffffff
|
|
|
|
|
QUAD $0x0000000000000000
|
|
|
|
|
QUAD $0x0000ffffffffffff
|
|
|
|
|
QUAD $0x0000000000000000
|
|
|
|
|
QUAD $0x00ffffffffffffff
|
|
|
|
|
QUAD $0x0000000000000000
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0x0000000000000000
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0x00000000000000ff
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0x000000000000ffff
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0x0000000000ffffff
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0x00000000ffffffff
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0x000000ffffffffff
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0x0000ffffffffffff
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0x00ffffffffffffff
|
|
|
|
|
|
|
|
|
|
// these are arguments to pshufb. They move data down from
|
|
|
|
|
// the high bytes of the register to the low bytes of the register.
|
|
|
|
|
// index is how many bytes to move.
|
|
|
|
|
TEXT shifts(SB),7,$0
|
|
|
|
|
QUAD $0x0000000000000000
|
|
|
|
|
QUAD $0x0000000000000000
|
|
|
|
|
QUAD $0xffffffffffffff0f
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0xffffffffffff0f0e
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0xffffffffff0f0e0d
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0xffffffff0f0e0d0c
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0xffffff0f0e0d0c0b
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0xffff0f0e0d0c0b0a
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0xff0f0e0d0c0b0a09
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0x0f0e0d0c0b0a0908
|
|
|
|
|
QUAD $0xffffffffffffffff
|
|
|
|
|
QUAD $0x0e0d0c0b0a090807
|
|
|
|
|
QUAD $0xffffffffffffff0f
|
|
|
|
|
QUAD $0x0d0c0b0a09080706
|
|
|
|
|
QUAD $0xffffffffffff0f0e
|
|
|
|
|
QUAD $0x0c0b0a0908070605
|
|
|
|
|
QUAD $0xffffffffff0f0e0d
|
|
|
|
|
QUAD $0x0b0a090807060504
|
|
|
|
|
QUAD $0xffffffff0f0e0d0c
|
|
|
|
|
QUAD $0x0a09080706050403
|
|
|
|
|
QUAD $0xffffff0f0e0d0c0b
|
|
|
|
|
QUAD $0x0908070605040302
|
|
|
|
|
QUAD $0xffff0f0e0d0c0b0a
|
|
|
|
|
QUAD $0x0807060504030201
|
|
|
|
|
QUAD $0xff0f0e0d0c0b0a09
|
2013-04-02 17:26:15 -06:00
|
|
|
|
|
|
|
|
TEXT runtime·memeq(SB),7,$0
|
|
|
|
|
MOVQ a+0(FP), SI
|
|
|
|
|
MOVQ b+8(FP), DI
|
|
|
|
|
MOVQ count+16(FP), BX
|
|
|
|
|
JMP runtime·memeqbody(SB)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
TEXT bytes·Equal(SB),7,$0
|
|
|
|
|
MOVQ a_len+8(FP), BX
|
|
|
|
|
MOVQ b_len+32(FP), CX
|
|
|
|
|
XORQ AX, AX
|
|
|
|
|
CMPQ BX, CX
|
|
|
|
|
JNE eqret
|
|
|
|
|
MOVQ a+0(FP), SI
|
|
|
|
|
MOVQ b+24(FP), DI
|
|
|
|
|
CALL runtime·memeqbody(SB)
|
|
|
|
|
eqret:
|
|
|
|
|
MOVB AX, ret+48(FP)
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
// a in SI
|
|
|
|
|
// b in DI
|
|
|
|
|
// count in BX
|
|
|
|
|
TEXT runtime·memeqbody(SB),7,$0
|
|
|
|
|
XORQ AX, AX
|
|
|
|
|
|
|
|
|
|
CMPQ BX, $8
|
|
|
|
|
JB small
|
|
|
|
|
|
|
|
|
|
// 64 bytes at a time using xmm registers
|
|
|
|
|
hugeloop:
|
|
|
|
|
CMPQ BX, $64
|
|
|
|
|
JB bigloop
|
|
|
|
|
MOVOU (SI), X0
|
|
|
|
|
MOVOU (DI), X1
|
|
|
|
|
MOVOU 16(SI), X2
|
|
|
|
|
MOVOU 16(DI), X3
|
|
|
|
|
MOVOU 32(SI), X4
|
|
|
|
|
MOVOU 32(DI), X5
|
|
|
|
|
MOVOU 48(SI), X6
|
|
|
|
|
MOVOU 48(DI), X7
|
|
|
|
|
PCMPEQB X1, X0
|
|
|
|
|
PCMPEQB X3, X2
|
|
|
|
|
PCMPEQB X5, X4
|
|
|
|
|
PCMPEQB X7, X6
|
|
|
|
|
PAND X2, X0
|
|
|
|
|
PAND X6, X4
|
|
|
|
|
PAND X4, X0
|
|
|
|
|
PMOVMSKB X0, DX
|
|
|
|
|
ADDQ $64, SI
|
|
|
|
|
ADDQ $64, DI
|
|
|
|
|
SUBQ $64, BX
|
|
|
|
|
CMPL DX, $0xffff
|
|
|
|
|
JEQ hugeloop
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
// 8 bytes at a time using 64-bit register
|
|
|
|
|
bigloop:
|
|
|
|
|
CMPQ BX, $8
|
|
|
|
|
JBE leftover
|
|
|
|
|
MOVQ (SI), CX
|
|
|
|
|
MOVQ (DI), DX
|
|
|
|
|
ADDQ $8, SI
|
|
|
|
|
ADDQ $8, DI
|
|
|
|
|
SUBQ $8, BX
|
|
|
|
|
CMPQ CX, DX
|
|
|
|
|
JEQ bigloop
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
// remaining 0-8 bytes
|
|
|
|
|
leftover:
|
|
|
|
|
MOVQ -8(SI)(BX*1), CX
|
|
|
|
|
MOVQ -8(DI)(BX*1), DX
|
|
|
|
|
CMPQ CX, DX
|
|
|
|
|
SETEQ AX
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
small:
|
|
|
|
|
CMPQ BX, $0
|
|
|
|
|
JEQ equal
|
|
|
|
|
|
|
|
|
|
LEAQ 0(BX*8), CX
|
|
|
|
|
NEGQ CX
|
|
|
|
|
|
|
|
|
|
CMPB SI, $0xf8
|
|
|
|
|
JA si_high
|
|
|
|
|
|
|
|
|
|
// load at SI won't cross a page boundary.
|
|
|
|
|
MOVQ (SI), SI
|
|
|
|
|
JMP si_finish
|
|
|
|
|
si_high:
|
|
|
|
|
// address ends in 11111xxx. Load up to bytes we want, move to correct position.
|
|
|
|
|
MOVQ -8(SI)(BX*1), SI
|
|
|
|
|
SHRQ CX, SI
|
|
|
|
|
si_finish:
|
|
|
|
|
|
|
|
|
|
// same for DI.
|
|
|
|
|
CMPB DI, $0xf8
|
|
|
|
|
JA di_high
|
|
|
|
|
MOVQ (DI), DI
|
|
|
|
|
JMP di_finish
|
|
|
|
|
di_high:
|
|
|
|
|
MOVQ -8(DI)(BX*1), DI
|
|
|
|
|
SHRQ CX, DI
|
|
|
|
|
di_finish:
|
|
|
|
|
|
|
|
|
|
SUBQ SI, DI
|
|
|
|
|
SHLQ CX, DI
|
|
|
|
|
equal:
|
|
|
|
|
SETEQ AX
|
|
|
|
|
RET
|
2013-05-14 17:05:51 -06:00
|
|
|
|
|
|
|
|
|
|
|
|
|
TEXT runtime·cmpstring(SB),7,$0
|
|
|
|
|
MOVQ s1+0(FP), SI
|
|
|
|
|
MOVQ s1+8(FP), BX
|
|
|
|
|
MOVQ s2+16(FP), DI
|
|
|
|
|
MOVQ s2+24(FP), DX
|
|
|
|
|
CALL runtime·cmpbody(SB)
|
|
|
|
|
MOVQ AX, res+32(FP)
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
TEXT bytes·Compare(SB),7,$0
|
|
|
|
|
MOVQ s1+0(FP), SI
|
|
|
|
|
MOVQ s1+8(FP), BX
|
|
|
|
|
MOVQ s2+24(FP), DI
|
|
|
|
|
MOVQ s2+32(FP), DX
|
|
|
|
|
CALL runtime·cmpbody(SB)
|
|
|
|
|
MOVQ AX, res+48(FP)
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
// input:
|
|
|
|
|
// SI = a
|
|
|
|
|
// DI = b
|
|
|
|
|
// BX = alen
|
|
|
|
|
// DX = blen
|
|
|
|
|
// output:
|
|
|
|
|
// AX = 1/0/-1
|
|
|
|
|
TEXT runtime·cmpbody(SB),7,$0
|
|
|
|
|
CMPQ SI, DI
|
|
|
|
|
JEQ cmp_allsame
|
|
|
|
|
CMPQ BX, DX
|
|
|
|
|
MOVQ DX, BP
|
|
|
|
|
CMOVQLT BX, BP // BP = min(alen, blen) = # of bytes to compare
|
|
|
|
|
CMPQ BP, $8
|
|
|
|
|
JB cmp_small
|
|
|
|
|
|
|
|
|
|
cmp_loop:
|
|
|
|
|
CMPQ BP, $16
|
|
|
|
|
JBE cmp_0through16
|
|
|
|
|
MOVOU (SI), X0
|
|
|
|
|
MOVOU (DI), X1
|
|
|
|
|
PCMPEQB X0, X1
|
|
|
|
|
PMOVMSKB X1, AX
|
|
|
|
|
XORQ $0xffff, AX // convert EQ to NE
|
|
|
|
|
JNE cmp_diff16 // branch if at least one byte is not equal
|
|
|
|
|
ADDQ $16, SI
|
|
|
|
|
ADDQ $16, DI
|
|
|
|
|
SUBQ $16, BP
|
|
|
|
|
JMP cmp_loop
|
|
|
|
|
|
|
|
|
|
// AX = bit mask of differences
|
|
|
|
|
cmp_diff16:
|
|
|
|
|
BSFQ AX, BX // index of first byte that differs
|
|
|
|
|
XORQ AX, AX
|
|
|
|
|
MOVB (SI)(BX*1), CX
|
|
|
|
|
CMPB CX, (DI)(BX*1)
|
|
|
|
|
SETHI AX
|
|
|
|
|
LEAQ -1(AX*2), AX // convert 1/0 to +1/-1
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
// 0 through 16 bytes left, alen>=8, blen>=8
|
|
|
|
|
cmp_0through16:
|
|
|
|
|
CMPQ BP, $8
|
|
|
|
|
JBE cmp_0through8
|
|
|
|
|
MOVQ (SI), AX
|
|
|
|
|
MOVQ (DI), CX
|
|
|
|
|
CMPQ AX, CX
|
|
|
|
|
JNE cmp_diff8
|
|
|
|
|
cmp_0through8:
|
|
|
|
|
MOVQ -8(SI)(BP*1), AX
|
|
|
|
|
MOVQ -8(DI)(BP*1), CX
|
|
|
|
|
CMPQ AX, CX
|
|
|
|
|
JEQ cmp_allsame
|
|
|
|
|
|
|
|
|
|
// AX and CX contain parts of a and b that differ.
|
|
|
|
|
cmp_diff8:
|
|
|
|
|
BSWAPQ AX // reverse order of bytes
|
|
|
|
|
BSWAPQ CX
|
|
|
|
|
XORQ AX, CX
|
|
|
|
|
BSRQ CX, CX // index of highest bit difference
|
|
|
|
|
SHRQ CX, AX // move a's bit to bottom
|
|
|
|
|
ANDQ $1, AX // mask bit
|
|
|
|
|
LEAQ -1(AX*2), AX // 1/0 => +1/-1
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
// 0-7 bytes in common
|
|
|
|
|
cmp_small:
|
|
|
|
|
LEAQ (BP*8), CX // bytes left -> bits left
|
|
|
|
|
NEGQ CX // - bits lift (== 64 - bits left mod 64)
|
|
|
|
|
JEQ cmp_allsame
|
|
|
|
|
|
|
|
|
|
// load bytes of a into high bytes of AX
|
|
|
|
|
CMPB SI, $0xf8
|
|
|
|
|
JA cmp_si_high
|
|
|
|
|
MOVQ (SI), SI
|
|
|
|
|
JMP cmp_si_finish
|
|
|
|
|
cmp_si_high:
|
|
|
|
|
MOVQ -8(SI)(BP*1), SI
|
|
|
|
|
SHRQ CX, SI
|
|
|
|
|
cmp_si_finish:
|
|
|
|
|
SHLQ CX, SI
|
|
|
|
|
|
|
|
|
|
// load bytes of b in to high bytes of BX
|
|
|
|
|
CMPB DI, $0xf8
|
|
|
|
|
JA cmp_di_high
|
|
|
|
|
MOVQ (DI), DI
|
|
|
|
|
JMP cmp_di_finish
|
|
|
|
|
cmp_di_high:
|
|
|
|
|
MOVQ -8(DI)(BP*1), DI
|
|
|
|
|
SHRQ CX, DI
|
|
|
|
|
cmp_di_finish:
|
|
|
|
|
SHLQ CX, DI
|
|
|
|
|
|
|
|
|
|
BSWAPQ SI // reverse order of bytes
|
|
|
|
|
BSWAPQ DI
|
|
|
|
|
XORQ SI, DI // find bit differences
|
|
|
|
|
JEQ cmp_allsame
|
|
|
|
|
BSRQ DI, CX // index of highest bit difference
|
|
|
|
|
SHRQ CX, SI // move a's bit to bottom
|
|
|
|
|
ANDQ $1, SI // mask bit
|
|
|
|
|
LEAQ -1(SI*2), AX // 1/0 => +1/-1
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
cmp_allsame:
|
|
|
|
|
XORQ AX, AX
|
|
|
|
|
XORQ CX, CX
|
|
|
|
|
CMPQ BX, DX
|
|
|
|
|
SETGT AX // 1 if alen > blen
|
|
|
|
|
SETEQ CX // 1 if alen == blen
|
|
|
|
|
LEAQ -1(CX)(AX*2), AX // 1,0,-1 result
|
|
|
|
|
RET
|
