2008-06-30 12:50:36 -06:00
|
|
|
// Copyright 2009 The Go Authors. All rights reserved.
|
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
[dev.cc] runtime: convert assembly files for C to Go transition
The main change is that #include "zasm_GOOS_GOARCH.h"
is now #include "go_asm.h" and/or #include "go_tls.h".
Also, because C StackGuard is now Go _StackGuard,
the assembly name changes from const_StackGuard to
const__StackGuard.
In asm_$GOARCH.s, add new function getg, formerly
implemented in C.
The renamed atomics now have Go wrappers, to get
escape analysis annotations right. Those wrappers
are in CL 174860043.
LGTM=r, aram
R=r, aram
CC=austin, dvyukov, golang-codereviews, iant, khr
https://golang.org/cl/168510043
2014-11-11 15:06:22 -07:00
|
|
|
#include "go_asm.h"
|
|
|
|
|
#include "go_tls.h"
|
2013-07-16 14:24:09 -06:00
|
|
|
#include "funcdata.h"
|
2014-09-04 21:05:18 -06:00
|
|
|
#include "textflag.h"
|
2008-06-30 12:50:36 -06:00
|
|
|
|
2014-09-03 09:11:16 -06:00
|
|
|
TEXT runtime·rt0_go(SB),NOSPLIT,$0
|
2008-06-30 12:50:36 -06:00
|
|
|
// copy arguments forward on an even stack
|
2013-03-06 13:03:04 -07:00
|
|
|
MOVQ DI, AX // argc
|
|
|
|
|
MOVQ SI, BX // argv
|
2008-06-30 12:50:36 -06:00
|
|
|
SUBQ $(4*8+7), SP // 2args 2auto
|
2010-04-09 15:15:15 -06:00
|
|
|
ANDQ $~15, SP
|
2008-06-30 12:50:36 -06:00
|
|
|
MOVQ AX, 16(SP)
|
|
|
|
|
MOVQ BX, 24(SP)
|
2011-12-07 06:53:17 -07:00
|
|
|
|
|
|
|
|
// create istack out of the given (operating system) stack.
|
2013-02-28 14:24:38 -07:00
|
|
|
// _cgo_init may update stackguard.
|
2011-12-07 06:53:17 -07:00
|
|
|
MOVQ $runtime·g0(SB), DI
|
2012-01-19 18:59:44 -07:00
|
|
|
LEAQ (-64*1024+104)(SP), BX
|
2015-01-05 09:29:21 -07:00
|
|
|
MOVQ BX, g_stackguard0(DI)
|
|
|
|
|
MOVQ BX, g_stackguard1(DI)
|
runtime: assume precisestack, copystack, StackCopyAlways, ScanStackByFrames
Commit to stack copying for stack growth.
We're carrying around a surprising amount of cruft from older schemes.
I am confident that precise stack scans and stack copying are here to stay.
Delete fallback code for when precise stack info is disabled.
Delete fallback code for when copying stacks is disabled.
Delete fallback code for when StackCopyAlways is disabled.
Delete Stktop chain - there is only one stack segment now.
Delete M.moreargp, M.moreargsize, M.moreframesize, M.cret.
Delete G.writenbuf (unrelated, just dead).
Delete runtime.lessstack, runtime.oldstack.
Delete many amd64 morestack variants.
Delete initialization of morestack frame/arg sizes (shortens split prologue!).
Replace G's stackguard/stackbase/stack0/stacksize/
syscallstack/syscallguard/forkstackguard with simple stack
bounds (lo, hi).
Update liblink, runtime/cgo for adjustments to G.
LGTM=khr
R=khr, bradfitz
CC=golang-codereviews, iant, r
https://golang.org/cl/137410043
2014-09-09 11:39:57 -06:00
|
|
|
MOVQ BX, (g_stack+stack_lo)(DI)
|
|
|
|
|
MOVQ SP, (g_stack+stack_hi)(DI)
|
2008-06-30 12:50:36 -06:00
|
|
|
|
2013-03-12 11:47:44 -06:00
|
|
|
// find out information about the processor we're on
|
|
|
|
|
MOVQ $0, AX
|
|
|
|
|
CPUID
|
2016-03-29 22:25:33 -06:00
|
|
|
MOVQ AX, SI
|
2013-03-12 11:47:44 -06:00
|
|
|
CMPQ AX, $0
|
|
|
|
|
JE nocpuinfo
|
2015-02-17 04:25:49 -07:00
|
|
|
|
|
|
|
|
// Figure out how to serialize RDTSC.
|
|
|
|
|
// On Intel processors LFENCE is enough. AMD requires MFENCE.
|
|
|
|
|
// Don't know about the rest, so let's do MFENCE.
|
|
|
|
|
CMPL BX, $0x756E6547 // "Genu"
|
|
|
|
|
JNE notintel
|
|
|
|
|
CMPL DX, $0x49656E69 // "ineI"
|
|
|
|
|
JNE notintel
|
|
|
|
|
CMPL CX, $0x6C65746E // "ntel"
|
|
|
|
|
JNE notintel
|
|
|
|
|
MOVB $1, runtime·lfenceBeforeRdtsc(SB)
|
|
|
|
|
notintel:
|
|
|
|
|
|
2016-03-29 22:25:33 -06:00
|
|
|
// Load EAX=1 cpuid flags
|
2013-03-12 11:47:44 -06:00
|
|
|
MOVQ $1, AX
|
|
|
|
|
CPUID
|
|
|
|
|
MOVL CX, runtime·cpuid_ecx(SB)
|
|
|
|
|
MOVL DX, runtime·cpuid_edx(SB)
|
2016-03-29 22:25:33 -06:00
|
|
|
|
|
|
|
|
// Load EAX=7/ECX=0 cpuid flags
|
|
|
|
|
CMPQ SI, $7
|
|
|
|
|
JLT no7
|
|
|
|
|
MOVL $7, AX
|
|
|
|
|
MOVL $0, CX
|
|
|
|
|
CPUID
|
|
|
|
|
MOVL BX, runtime·cpuid_ebx7(SB)
|
|
|
|
|
no7:
|
2015-10-28 14:20:26 -06:00
|
|
|
// Detect AVX and AVX2 as per 14.7.1 Detection of AVX2 chapter of [1]
|
|
|
|
|
// [1] 64-ia-32-architectures-software-developer-manual-325462.pdf
|
|
|
|
|
// http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-manual-325462.pdf
|
2016-03-29 22:25:33 -06:00
|
|
|
MOVL runtime·cpuid_ecx(SB), CX
|
2015-10-28 14:20:26 -06:00
|
|
|
ANDL $0x18000000, CX // check for OSXSAVE and AVX bits
|
|
|
|
|
CMPL CX, $0x18000000
|
|
|
|
|
JNE noavx
|
|
|
|
|
MOVL $0, CX
|
|
|
|
|
// For XGETBV, OSXSAVE bit is required and sufficient
|
2016-01-13 06:43:22 -07:00
|
|
|
XGETBV
|
2015-10-28 14:20:26 -06:00
|
|
|
ANDL $6, AX
|
|
|
|
|
CMPL AX, $6 // Check for OS support of YMM registers
|
|
|
|
|
JNE noavx
|
|
|
|
|
MOVB $1, runtime·support_avx(SB)
|
2016-03-29 22:25:33 -06:00
|
|
|
TESTL $(1<<5), runtime·cpuid_ebx7(SB) // check for AVX2 bit
|
|
|
|
|
JEQ noavx2
|
2015-10-28 14:20:26 -06:00
|
|
|
MOVB $1, runtime·support_avx2(SB)
|
|
|
|
|
JMP nocpuinfo
|
|
|
|
|
noavx:
|
|
|
|
|
MOVB $0, runtime·support_avx(SB)
|
|
|
|
|
noavx2:
|
|
|
|
|
MOVB $0, runtime·support_avx2(SB)
|
2013-03-12 11:47:44 -06:00
|
|
|
nocpuinfo:
|
|
|
|
|
|
2013-02-28 14:24:38 -07:00
|
|
|
// if there is an _cgo_init, call it.
|
|
|
|
|
MOVQ _cgo_init(SB), AX
|
2010-04-09 15:15:15 -06:00
|
|
|
TESTQ AX, AX
|
2010-08-04 18:50:22 -06:00
|
|
|
JZ needtls
|
2012-01-19 18:59:44 -07:00
|
|
|
// g0 already in DI
|
|
|
|
|
MOVQ DI, CX // Win64 uses CX for first parameter
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
MOVQ $setg_gcc<>(SB), SI
|
2012-01-19 18:59:44 -07:00
|
|
|
CALL AX
|
runtime: assume precisestack, copystack, StackCopyAlways, ScanStackByFrames
Commit to stack copying for stack growth.
We're carrying around a surprising amount of cruft from older schemes.
I am confident that precise stack scans and stack copying are here to stay.
Delete fallback code for when precise stack info is disabled.
Delete fallback code for when copying stacks is disabled.
Delete fallback code for when StackCopyAlways is disabled.
Delete Stktop chain - there is only one stack segment now.
Delete M.moreargp, M.moreargsize, M.moreframesize, M.cret.
Delete G.writenbuf (unrelated, just dead).
Delete runtime.lessstack, runtime.oldstack.
Delete many amd64 morestack variants.
Delete initialization of morestack frame/arg sizes (shortens split prologue!).
Replace G's stackguard/stackbase/stack0/stacksize/
syscallstack/syscallguard/forkstackguard with simple stack
bounds (lo, hi).
Update liblink, runtime/cgo for adjustments to G.
LGTM=khr
R=khr, bradfitz
CC=golang-codereviews, iant, r
https://golang.org/cl/137410043
2014-09-09 11:39:57 -06:00
|
|
|
|
2013-06-03 02:28:24 -06:00
|
|
|
// update stackguard after _cgo_init
|
|
|
|
|
MOVQ $runtime·g0(SB), CX
|
runtime: assume precisestack, copystack, StackCopyAlways, ScanStackByFrames
Commit to stack copying for stack growth.
We're carrying around a surprising amount of cruft from older schemes.
I am confident that precise stack scans and stack copying are here to stay.
Delete fallback code for when precise stack info is disabled.
Delete fallback code for when copying stacks is disabled.
Delete fallback code for when StackCopyAlways is disabled.
Delete Stktop chain - there is only one stack segment now.
Delete M.moreargp, M.moreargsize, M.moreframesize, M.cret.
Delete G.writenbuf (unrelated, just dead).
Delete runtime.lessstack, runtime.oldstack.
Delete many amd64 morestack variants.
Delete initialization of morestack frame/arg sizes (shortens split prologue!).
Replace G's stackguard/stackbase/stack0/stacksize/
syscallstack/syscallguard/forkstackguard with simple stack
bounds (lo, hi).
Update liblink, runtime/cgo for adjustments to G.
LGTM=khr
R=khr, bradfitz
CC=golang-codereviews, iant, r
https://golang.org/cl/137410043
2014-09-09 11:39:57 -06:00
|
|
|
MOVQ (g_stack+stack_lo)(CX), AX
|
[dev.cc] runtime: convert assembly files for C to Go transition
The main change is that #include "zasm_GOOS_GOARCH.h"
is now #include "go_asm.h" and/or #include "go_tls.h".
Also, because C StackGuard is now Go _StackGuard,
the assembly name changes from const_StackGuard to
const__StackGuard.
In asm_$GOARCH.s, add new function getg, formerly
implemented in C.
The renamed atomics now have Go wrappers, to get
escape analysis annotations right. Those wrappers
are in CL 174860043.
LGTM=r, aram
R=r, aram
CC=austin, dvyukov, golang-codereviews, iant, khr
https://golang.org/cl/168510043
2014-11-11 15:06:22 -07:00
|
|
|
ADDQ $const__StackGuard, AX
|
2015-01-05 09:29:21 -07:00
|
|
|
MOVQ AX, g_stackguard0(CX)
|
|
|
|
|
MOVQ AX, g_stackguard1(CX)
|
runtime: assume precisestack, copystack, StackCopyAlways, ScanStackByFrames
Commit to stack copying for stack growth.
We're carrying around a surprising amount of cruft from older schemes.
I am confident that precise stack scans and stack copying are here to stay.
Delete fallback code for when precise stack info is disabled.
Delete fallback code for when copying stacks is disabled.
Delete fallback code for when StackCopyAlways is disabled.
Delete Stktop chain - there is only one stack segment now.
Delete M.moreargp, M.moreargsize, M.moreframesize, M.cret.
Delete G.writenbuf (unrelated, just dead).
Delete runtime.lessstack, runtime.oldstack.
Delete many amd64 morestack variants.
Delete initialization of morestack frame/arg sizes (shortens split prologue!).
Replace G's stackguard/stackbase/stack0/stacksize/
syscallstack/syscallguard/forkstackguard with simple stack
bounds (lo, hi).
Update liblink, runtime/cgo for adjustments to G.
LGTM=khr
R=khr, bradfitz
CC=golang-codereviews, iant, r
https://golang.org/cl/137410043
2014-09-09 11:39:57 -06:00
|
|
|
|
2015-06-09 16:24:38 -06:00
|
|
|
#ifndef GOOS_windows
|
|
|
|
|
JMP ok
|
|
|
|
|
#endif
|
2010-08-04 18:50:22 -06:00
|
|
|
needtls:
|
2015-06-09 16:24:38 -06:00
|
|
|
#ifdef GOOS_plan9
|
2012-08-31 11:21:13 -06:00
|
|
|
// skip TLS setup on Plan 9
|
2015-06-09 16:24:38 -06:00
|
|
|
JMP ok
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef GOOS_solaris
|
2014-01-16 21:58:10 -07:00
|
|
|
// skip TLS setup on Solaris
|
2015-06-09 16:24:38 -06:00
|
|
|
JMP ok
|
|
|
|
|
#endif
|
2012-08-31 11:21:13 -06:00
|
|
|
|
2015-11-12 16:35:50 -07:00
|
|
|
LEAQ runtime·m0+m_tls(SB), DI
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
CALL runtime·settls(SB)
|
2010-08-04 18:50:22 -06:00
|
|
|
|
|
|
|
|
// store through it, to make sure it works
|
|
|
|
|
get_tls(BX)
|
|
|
|
|
MOVQ $0x123, g(BX)
|
2015-11-12 16:35:50 -07:00
|
|
|
MOVQ runtime·m0+m_tls(SB), AX
|
2010-08-04 18:50:22 -06:00
|
|
|
CMPQ AX, $0x123
|
|
|
|
|
JEQ 2(PC)
|
|
|
|
|
MOVL AX, 0 // abort
|
|
|
|
|
ok:
|
|
|
|
|
// set the per-goroutine and per-mach "registers"
|
|
|
|
|
get_tls(BX)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
LEAQ runtime·g0(SB), CX
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ CX, g(BX)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
LEAQ runtime·m0(SB), AX
|
2010-08-04 18:50:22 -06:00
|
|
|
|
|
|
|
|
// save m->g0 = g0
|
|
|
|
|
MOVQ CX, m_g0(AX)
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
// save m0 to g0->m
|
|
|
|
|
MOVQ AX, g_m(CX)
|
2008-06-30 12:50:36 -06:00
|
|
|
|
2008-12-15 16:07:35 -07:00
|
|
|
CLD // convention is D is always left cleared
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
CALL runtime·check(SB)
|
2008-06-30 12:50:36 -06:00
|
|
|
|
|
|
|
|
MOVL 16(SP), AX // copy argc
|
|
|
|
|
MOVL AX, 0(SP)
|
|
|
|
|
MOVQ 24(SP), AX // copy argv
|
|
|
|
|
MOVQ AX, 8(SP)
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
CALL runtime·args(SB)
|
|
|
|
|
CALL runtime·osinit(SB)
|
|
|
|
|
CALL runtime·schedinit(SB)
|
2008-08-05 15:21:42 -06:00
|
|
|
|
2008-07-11 20:16:39 -06:00
|
|
|
// create a new goroutine to start program
|
2015-03-29 17:38:20 -06:00
|
|
|
MOVQ $runtime·mainPC(SB), AX // entry
|
2015-01-27 16:29:02 -07:00
|
|
|
PUSHQ AX
|
2009-06-17 16:12:16 -06:00
|
|
|
PUSHQ $0 // arg size
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
CALL runtime·newproc(SB)
|
2008-07-11 20:16:39 -06:00
|
|
|
POPQ AX
|
|
|
|
|
POPQ AX
|
2008-12-04 09:30:54 -07:00
|
|
|
|
2008-09-22 14:47:59 -06:00
|
|
|
// start this M
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
CALL runtime·mstart(SB)
|
2008-06-30 12:50:36 -06:00
|
|
|
|
2012-03-08 12:03:56 -07:00
|
|
|
MOVL $0xf1, 0xf1 // crash
|
2008-06-30 12:50:36 -06:00
|
|
|
RET
|
|
|
|
|
|
2015-03-29 17:38:20 -06:00
|
|
|
DATA runtime·mainPC+0(SB)/8,$runtime·main(SB)
|
|
|
|
|
GLOBL runtime·mainPC(SB),RODATA,$8
|
2013-02-21 15:01:13 -07:00
|
|
|
|
2013-08-07 11:23:24 -06:00
|
|
|
TEXT runtime·breakpoint(SB),NOSPLIT,$0-0
|
2008-07-11 20:16:39 -06:00
|
|
|
BYTE $0xcc
|
2008-06-30 12:50:36 -06:00
|
|
|
RET
|
|
|
|
|
|
2013-08-07 11:23:24 -06:00
|
|
|
TEXT runtime·asminit(SB),NOSPLIT,$0-0
|
2012-02-13 23:23:15 -07:00
|
|
|
// No per-thread init.
|
|
|
|
|
RET
|
|
|
|
|
|
2008-07-11 20:16:39 -06:00
|
|
|
/*
|
|
|
|
|
* go-routine
|
|
|
|
|
*/
|
2008-07-07 18:59:32 -06:00
|
|
|
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// void gosave(Gobuf*)
|
2009-06-17 16:12:16 -06:00
|
|
|
// save state in Gobuf; setjmp
|
2013-08-07 11:23:24 -06:00
|
|
|
TEXT runtime·gosave(SB), NOSPLIT, $0-8
|
cmd/cc, runtime: convert C compilers to use Go calling convention
To date, the C compilers and Go compilers differed only in how
values were returned from functions. This made it difficult to call
Go from C or C from Go if return values were involved. It also made
assembly called from Go and assembly called from C different.
This CL changes the C compiler to use the Go conventions, passing
results on the stack, after the arguments.
[Exception: this does not apply to C ... functions, because you can't
know where on the stack the arguments end.]
By doing this, the CL makes it possible to rewrite C functions into Go
one at a time, without worrying about which languages call that
function or which languages it calls.
This CL also updates all the assembly files in package runtime to use
the new conventions. Argument references of the form 40(SP) have
been rewritten to the form name+10(FP) instead, and there are now
Go func prototypes for every assembly function called from C or Go.
This means that 'go vet runtime' checks effectively every assembly
function, and go vet's output was used to automate the bulk of the
conversion.
Some functions, like seek and nsec on Plan 9, needed to be rewritten.
Many assembly routines called from C were reading arguments
incorrectly, using MOVL instead of MOVQ or vice versa, especially on
the less used systems like openbsd.
These were found by go vet and have been corrected too.
If we're lucky, this may reduce flakiness on those systems.
Tested on:
darwin/386
darwin/amd64
linux/arm
linux/386
linux/amd64
If this breaks another system, the bug is almost certainly in the
sys_$GOOS_$GOARCH.s file, since the rest of the CL is tested
by the combination of the above systems.
LGTM=dvyukov, iant
R=golang-codereviews, 0intro, dave, alex.brainman, dvyukov, iant
CC=golang-codereviews, josharian, r
https://golang.org/cl/135830043
2014-08-27 09:32:17 -06:00
|
|
|
MOVQ buf+0(FP), AX // gobuf
|
|
|
|
|
LEAQ buf+0(FP), BX // caller's SP
|
2009-06-17 16:12:16 -06:00
|
|
|
MOVQ BX, gobuf_sp(AX)
|
|
|
|
|
MOVQ 0(SP), BX // caller's PC
|
|
|
|
|
MOVQ BX, gobuf_pc(AX)
|
2013-06-12 13:22:26 -06:00
|
|
|
MOVQ $0, gobuf_ret(AX)
|
|
|
|
|
MOVQ $0, gobuf_ctxt(AX)
|
2015-01-14 09:09:50 -07:00
|
|
|
MOVQ BP, gobuf_bp(AX)
|
2010-08-04 18:50:22 -06:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ g(CX), BX
|
|
|
|
|
MOVQ BX, gobuf_g(AX)
|
2008-06-30 12:50:36 -06:00
|
|
|
RET
|
|
|
|
|
|
2013-06-12 16:05:10 -06:00
|
|
|
// void gogo(Gobuf*)
|
2009-06-17 16:12:16 -06:00
|
|
|
// restore state from Gobuf; longjmp
|
2013-08-07 11:23:24 -06:00
|
|
|
TEXT runtime·gogo(SB), NOSPLIT, $0-8
|
cmd/cc, runtime: convert C compilers to use Go calling convention
To date, the C compilers and Go compilers differed only in how
values were returned from functions. This made it difficult to call
Go from C or C from Go if return values were involved. It also made
assembly called from Go and assembly called from C different.
This CL changes the C compiler to use the Go conventions, passing
results on the stack, after the arguments.
[Exception: this does not apply to C ... functions, because you can't
know where on the stack the arguments end.]
By doing this, the CL makes it possible to rewrite C functions into Go
one at a time, without worrying about which languages call that
function or which languages it calls.
This CL also updates all the assembly files in package runtime to use
the new conventions. Argument references of the form 40(SP) have
been rewritten to the form name+10(FP) instead, and there are now
Go func prototypes for every assembly function called from C or Go.
This means that 'go vet runtime' checks effectively every assembly
function, and go vet's output was used to automate the bulk of the
conversion.
Some functions, like seek and nsec on Plan 9, needed to be rewritten.
Many assembly routines called from C were reading arguments
incorrectly, using MOVL instead of MOVQ or vice versa, especially on
the less used systems like openbsd.
These were found by go vet and have been corrected too.
If we're lucky, this may reduce flakiness on those systems.
Tested on:
darwin/386
darwin/amd64
linux/arm
linux/386
linux/amd64
If this breaks another system, the bug is almost certainly in the
sys_$GOOS_$GOARCH.s file, since the rest of the CL is tested
by the combination of the above systems.
LGTM=dvyukov, iant
R=golang-codereviews, 0intro, dave, alex.brainman, dvyukov, iant
CC=golang-codereviews, josharian, r
https://golang.org/cl/135830043
2014-08-27 09:32:17 -06:00
|
|
|
MOVQ buf+0(FP), BX // gobuf
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ gobuf_g(BX), DX
|
|
|
|
|
MOVQ 0(DX), CX // make sure g != nil
|
|
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ DX, g(CX)
|
2009-06-17 16:12:16 -06:00
|
|
|
MOVQ gobuf_sp(BX), SP // restore SP
|
2013-06-12 13:22:26 -06:00
|
|
|
MOVQ gobuf_ret(BX), AX
|
|
|
|
|
MOVQ gobuf_ctxt(BX), DX
|
2015-01-14 09:09:50 -07:00
|
|
|
MOVQ gobuf_bp(BX), BP
|
2013-06-12 13:22:26 -06:00
|
|
|
MOVQ $0, gobuf_sp(BX) // clear to help garbage collector
|
|
|
|
|
MOVQ $0, gobuf_ret(BX)
|
|
|
|
|
MOVQ $0, gobuf_ctxt(BX)
|
2015-01-14 09:09:50 -07:00
|
|
|
MOVQ $0, gobuf_bp(BX)
|
2009-06-17 16:12:16 -06:00
|
|
|
MOVQ gobuf_pc(BX), BX
|
|
|
|
|
JMP BX
|
|
|
|
|
|
2014-09-03 09:35:22 -06:00
|
|
|
// func mcall(fn func(*g))
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// Switch to m->g0's stack, call fn(g).
|
2016-03-01 16:21:55 -07:00
|
|
|
// Fn must never return. It should gogo(&g->sched)
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// to keep running g.
|
2013-08-07 11:23:24 -06:00
|
|
|
TEXT runtime·mcall(SB), NOSPLIT, $0-8
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ fn+0(FP), DI
|
|
|
|
|
|
|
|
|
|
get_tls(CX)
|
2013-06-05 05:16:53 -06:00
|
|
|
MOVQ g(CX), AX // save state in g->sched
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ 0(SP), BX // caller's PC
|
|
|
|
|
MOVQ BX, (g_sched+gobuf_pc)(AX)
|
cmd/cc, runtime: convert C compilers to use Go calling convention
To date, the C compilers and Go compilers differed only in how
values were returned from functions. This made it difficult to call
Go from C or C from Go if return values were involved. It also made
assembly called from Go and assembly called from C different.
This CL changes the C compiler to use the Go conventions, passing
results on the stack, after the arguments.
[Exception: this does not apply to C ... functions, because you can't
know where on the stack the arguments end.]
By doing this, the CL makes it possible to rewrite C functions into Go
one at a time, without worrying about which languages call that
function or which languages it calls.
This CL also updates all the assembly files in package runtime to use
the new conventions. Argument references of the form 40(SP) have
been rewritten to the form name+10(FP) instead, and there are now
Go func prototypes for every assembly function called from C or Go.
This means that 'go vet runtime' checks effectively every assembly
function, and go vet's output was used to automate the bulk of the
conversion.
Some functions, like seek and nsec on Plan 9, needed to be rewritten.
Many assembly routines called from C were reading arguments
incorrectly, using MOVL instead of MOVQ or vice versa, especially on
the less used systems like openbsd.
These were found by go vet and have been corrected too.
If we're lucky, this may reduce flakiness on those systems.
Tested on:
darwin/386
darwin/amd64
linux/arm
linux/386
linux/amd64
If this breaks another system, the bug is almost certainly in the
sys_$GOOS_$GOARCH.s file, since the rest of the CL is tested
by the combination of the above systems.
LGTM=dvyukov, iant
R=golang-codereviews, 0intro, dave, alex.brainman, dvyukov, iant
CC=golang-codereviews, josharian, r
https://golang.org/cl/135830043
2014-08-27 09:32:17 -06:00
|
|
|
LEAQ fn+0(FP), BX // caller's SP
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ BX, (g_sched+gobuf_sp)(AX)
|
|
|
|
|
MOVQ AX, (g_sched+gobuf_g)(AX)
|
2015-01-14 09:09:50 -07:00
|
|
|
MOVQ BP, (g_sched+gobuf_bp)(AX)
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
|
|
|
|
|
// switch to m->g0 & its stack, call fn
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
MOVQ g(CX), BX
|
|
|
|
|
MOVQ g_m(BX), BX
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ m_g0(BX), SI
|
|
|
|
|
CMPQ SI, AX // if g == m->g0 call badmcall
|
2013-07-16 14:24:09 -06:00
|
|
|
JNE 3(PC)
|
2013-08-29 16:53:34 -06:00
|
|
|
MOVQ $runtime·badmcall(SB), AX
|
|
|
|
|
JMP AX
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ SI, g(CX) // g = m->g0
|
2013-06-05 05:16:53 -06:00
|
|
|
MOVQ (g_sched+gobuf_sp)(SI), SP // sp = m->g0->sched.sp
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
PUSHQ AX
|
2014-09-03 09:35:22 -06:00
|
|
|
MOVQ DI, DX
|
|
|
|
|
MOVQ 0(DI), DI
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
CALL DI
|
|
|
|
|
POPQ AX
|
2013-08-29 16:53:34 -06:00
|
|
|
MOVQ $runtime·badmcall2(SB), AX
|
|
|
|
|
JMP AX
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
RET
|
|
|
|
|
|
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://golang.org/cl/174950043
2014-11-12 12:54:31 -07:00
|
|
|
// systemstack_switch is a dummy routine that systemstack leaves at the bottom
|
2016-03-01 16:21:55 -07:00
|
|
|
// of the G stack. We need to distinguish the routine that
|
2014-07-30 10:01:52 -06:00
|
|
|
// lives at the bottom of the G stack from the one that lives
|
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://golang.org/cl/174950043
2014-11-12 12:54:31 -07:00
|
|
|
// at the top of the system stack because the one at the top of
|
|
|
|
|
// the system stack terminates the stack walk (see topofstack()).
|
|
|
|
|
TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
|
2014-07-30 10:01:52 -06:00
|
|
|
RET
|
|
|
|
|
|
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://golang.org/cl/174950043
2014-11-12 12:54:31 -07:00
|
|
|
// func systemstack(fn func())
|
|
|
|
|
TEXT runtime·systemstack(SB), NOSPLIT, $0-8
|
|
|
|
|
MOVQ fn+0(FP), DI // DI = fn
|
2014-09-11 10:08:30 -06:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ g(CX), AX // AX = g
|
|
|
|
|
MOVQ g_m(AX), BX // BX = m
|
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://golang.org/cl/174950043
2014-11-12 12:54:31 -07:00
|
|
|
|
2014-09-11 10:08:30 -06:00
|
|
|
MOVQ m_gsignal(BX), DX // DX = gsignal
|
|
|
|
|
CMPQ AX, DX
|
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://golang.org/cl/174950043
2014-11-12 12:54:31 -07:00
|
|
|
JEQ noswitch
|
runtime: reject onM calls from gsignal stack
The implementation and use patterns of onM assume
that they run on either the m->curg or m->g0 stack.
Calling onM from m->gsignal has two problems:
(1) When not on g0, onM switches to g0 and then "back" to curg.
If we didn't start at curg, bad things happen.
(2) The use of scalararg/ptrarg to pass C arguments and results
assumes that there is only one onM call at a time.
If a gsignal starts running, it may have interrupted the
setup/teardown of the args for an onM on the curg or g0 stack.
Using scalararg/ptrarg itself would smash those.
We can fix (1) by remembering what g was running before the switch.
We can fix (2) by requiring that uses of onM that might happen
on a signal handling stack must save the old scalararg/ptrarg
and restore them after the call, instead of zeroing them.
The only sane way to do this is to introduce a separate
onM_signalsafe that omits the signal check, and then if you
see a call to onM_signalsafe you know the surrounding code
must preserve the old scalararg/ptrarg values.
(The implementation would be that onM_signalsafe just calls
fn if on the signal stack or else jumps to onM. It's not necessary
to have two whole copies of the function.)
(2) is not a problem if the caller and callee are both Go and
a closure is used instead of the scalararg/ptrarg slots.
For now, I think we can avoid calling onM from code executing
on gsignal stacks, so just reject it.
In the long term, (2) goes away (as do the scalararg/ptrarg slots)
once everything is in Go, and at that point fixing (1) would be
trivial and maybe worth doing just for regularity.
LGTM=iant
R=golang-codereviews, iant
CC=dvyukov, golang-codereviews, khr, r
https://golang.org/cl/135400043
2014-09-03 22:10:10 -06:00
|
|
|
|
2014-07-30 10:01:52 -06:00
|
|
|
MOVQ m_g0(BX), DX // DX = g0
|
|
|
|
|
CMPQ AX, DX
|
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://golang.org/cl/174950043
2014-11-12 12:54:31 -07:00
|
|
|
JEQ noswitch
|
2014-07-30 10:01:52 -06:00
|
|
|
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ m_curg(BX), R8
|
|
|
|
|
CMPQ AX, R8
|
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://golang.org/cl/174950043
2014-11-12 12:54:31 -07:00
|
|
|
JEQ switch
|
runtime: reject onM calls from gsignal stack
The implementation and use patterns of onM assume
that they run on either the m->curg or m->g0 stack.
Calling onM from m->gsignal has two problems:
(1) When not on g0, onM switches to g0 and then "back" to curg.
If we didn't start at curg, bad things happen.
(2) The use of scalararg/ptrarg to pass C arguments and results
assumes that there is only one onM call at a time.
If a gsignal starts running, it may have interrupted the
setup/teardown of the args for an onM on the curg or g0 stack.
Using scalararg/ptrarg itself would smash those.
We can fix (1) by remembering what g was running before the switch.
We can fix (2) by requiring that uses of onM that might happen
on a signal handling stack must save the old scalararg/ptrarg
and restore them after the call, instead of zeroing them.
The only sane way to do this is to introduce a separate
onM_signalsafe that omits the signal check, and then if you
see a call to onM_signalsafe you know the surrounding code
must preserve the old scalararg/ptrarg values.
(The implementation would be that onM_signalsafe just calls
fn if on the signal stack or else jumps to onM. It's not necessary
to have two whole copies of the function.)
(2) is not a problem if the caller and callee are both Go and
a closure is used instead of the scalararg/ptrarg slots.
For now, I think we can avoid calling onM from code executing
on gsignal stacks, so just reject it.
In the long term, (2) goes away (as do the scalararg/ptrarg slots)
once everything is in Go, and at that point fixing (1) would be
trivial and maybe worth doing just for regularity.
LGTM=iant
R=golang-codereviews, iant
CC=dvyukov, golang-codereviews, khr, r
https://golang.org/cl/135400043
2014-09-03 22:10:10 -06:00
|
|
|
|
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://golang.org/cl/174950043
2014-11-12 12:54:31 -07:00
|
|
|
// Bad: g is not gsignal, not g0, not curg. What is it?
|
|
|
|
|
MOVQ $runtime·badsystemstack(SB), AX
|
runtime: reject onM calls from gsignal stack
The implementation and use patterns of onM assume
that they run on either the m->curg or m->g0 stack.
Calling onM from m->gsignal has two problems:
(1) When not on g0, onM switches to g0 and then "back" to curg.
If we didn't start at curg, bad things happen.
(2) The use of scalararg/ptrarg to pass C arguments and results
assumes that there is only one onM call at a time.
If a gsignal starts running, it may have interrupted the
setup/teardown of the args for an onM on the curg or g0 stack.
Using scalararg/ptrarg itself would smash those.
We can fix (1) by remembering what g was running before the switch.
We can fix (2) by requiring that uses of onM that might happen
on a signal handling stack must save the old scalararg/ptrarg
and restore them after the call, instead of zeroing them.
The only sane way to do this is to introduce a separate
onM_signalsafe that omits the signal check, and then if you
see a call to onM_signalsafe you know the surrounding code
must preserve the old scalararg/ptrarg values.
(The implementation would be that onM_signalsafe just calls
fn if on the signal stack or else jumps to onM. It's not necessary
to have two whole copies of the function.)
(2) is not a problem if the caller and callee are both Go and
a closure is used instead of the scalararg/ptrarg slots.
For now, I think we can avoid calling onM from code executing
on gsignal stacks, so just reject it.
In the long term, (2) goes away (as do the scalararg/ptrarg slots)
once everything is in Go, and at that point fixing (1) would be
trivial and maybe worth doing just for regularity.
LGTM=iant
R=golang-codereviews, iant
CC=dvyukov, golang-codereviews, khr, r
https://golang.org/cl/135400043
2014-09-03 22:10:10 -06:00
|
|
|
CALL AX
|
|
|
|
|
|
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://golang.org/cl/174950043
2014-11-12 12:54:31 -07:00
|
|
|
switch:
|
2016-03-01 16:21:55 -07:00
|
|
|
// save our state in g->sched. Pretend to
|
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://golang.org/cl/174950043
2014-11-12 12:54:31 -07:00
|
|
|
// be systemstack_switch if the G stack is scanned.
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ $runtime·systemstack_switch(SB), SI
|
|
|
|
|
MOVQ SI, (g_sched+gobuf_pc)(AX)
|
2014-07-30 10:01:52 -06:00
|
|
|
MOVQ SP, (g_sched+gobuf_sp)(AX)
|
|
|
|
|
MOVQ AX, (g_sched+gobuf_g)(AX)
|
2015-01-14 09:09:50 -07:00
|
|
|
MOVQ BP, (g_sched+gobuf_bp)(AX)
|
2014-07-30 10:01:52 -06:00
|
|
|
|
|
|
|
|
// switch to g0
|
|
|
|
|
MOVQ DX, g(CX)
|
runtime: do not stop traceback at onM
Behavior before this CL:
1. If onM is called on a g0 stack, it just calls the given function.
2. If onM is called on a gsignal stack, it calls badonm.
3. If onM is called on a curg stack, it switches to the g0 stack
and then calls the function.
In cases 1 and 2, if the program then crashes (and badonm always does),
we want to see what called onM, but the traceback stops at onM.
In case 3, the traceback must stop at onM, because the g0
stack we are renting really does stop at onM.
The current code stops the traceback at onM to handle 3,
at the cost of making 1 and 2 crash with incomplete traces.
Change traceback to scan past onM but in case 3 make it look
like on the rented g0 stack, onM was called from mstart.
The traceback already knows that mstart is a top-of-stack function.
Alternate fix at CL 132610043 but I think this one is cleaner.
This CL makes 3 the exception, while that CL makes 1 and 2 the exception.
Submitting TBR to try to get better stack traces out of the
freebsd/amd64 builder, but happy to make changes in a
followup CL.
TBR=khr
R=khr
CC=golang-codereviews
https://golang.org/cl/133620043
2014-09-04 20:48:08 -06:00
|
|
|
MOVQ (g_sched+gobuf_sp)(DX), BX
|
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://golang.org/cl/174950043
2014-11-12 12:54:31 -07:00
|
|
|
// make it look like mstart called systemstack on g0, to stop traceback
|
runtime: do not stop traceback at onM
Behavior before this CL:
1. If onM is called on a g0 stack, it just calls the given function.
2. If onM is called on a gsignal stack, it calls badonm.
3. If onM is called on a curg stack, it switches to the g0 stack
and then calls the function.
In cases 1 and 2, if the program then crashes (and badonm always does),
we want to see what called onM, but the traceback stops at onM.
In case 3, the traceback must stop at onM, because the g0
stack we are renting really does stop at onM.
The current code stops the traceback at onM to handle 3,
at the cost of making 1 and 2 crash with incomplete traces.
Change traceback to scan past onM but in case 3 make it look
like on the rented g0 stack, onM was called from mstart.
The traceback already knows that mstart is a top-of-stack function.
Alternate fix at CL 132610043 but I think this one is cleaner.
This CL makes 3 the exception, while that CL makes 1 and 2 the exception.
Submitting TBR to try to get better stack traces out of the
freebsd/amd64 builder, but happy to make changes in a
followup CL.
TBR=khr
R=khr
CC=golang-codereviews
https://golang.org/cl/133620043
2014-09-04 20:48:08 -06:00
|
|
|
SUBQ $8, BX
|
|
|
|
|
MOVQ $runtime·mstart(SB), DX
|
|
|
|
|
MOVQ DX, 0(BX)
|
|
|
|
|
MOVQ BX, SP
|
2014-07-30 10:01:52 -06:00
|
|
|
|
|
|
|
|
// call target function
|
2014-09-03 09:35:22 -06:00
|
|
|
MOVQ DI, DX
|
|
|
|
|
MOVQ 0(DI), DI
|
2014-07-30 10:01:52 -06:00
|
|
|
CALL DI
|
|
|
|
|
|
|
|
|
|
// switch back to g
|
|
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ g(CX), AX
|
|
|
|
|
MOVQ g_m(AX), BX
|
|
|
|
|
MOVQ m_curg(BX), AX
|
|
|
|
|
MOVQ AX, g(CX)
|
|
|
|
|
MOVQ (g_sched+gobuf_sp)(AX), SP
|
|
|
|
|
MOVQ $0, (g_sched+gobuf_sp)(AX)
|
|
|
|
|
RET
|
|
|
|
|
|
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://golang.org/cl/174950043
2014-11-12 12:54:31 -07:00
|
|
|
noswitch:
|
2014-07-30 10:01:52 -06:00
|
|
|
// already on m stack, just call directly
|
2014-09-03 09:35:22 -06:00
|
|
|
MOVQ DI, DX
|
|
|
|
|
MOVQ 0(DI), DI
|
2014-07-30 10:01:52 -06:00
|
|
|
CALL DI
|
|
|
|
|
RET
|
|
|
|
|
|
2008-07-12 12:30:53 -06:00
|
|
|
/*
|
|
|
|
|
* support for morestack
|
|
|
|
|
*/
|
|
|
|
|
|
2009-06-17 16:12:16 -06:00
|
|
|
// Called during function prolog when more stack is needed.
|
2013-07-18 14:53:45 -06:00
|
|
|
//
|
|
|
|
|
// The traceback routines see morestack on a g0 as being
|
|
|
|
|
// the top of a stack (for example, morestack calling newstack
|
|
|
|
|
// calling the scheduler calling newm calling gc), so we must
|
|
|
|
|
// record an argument size. For that purpose, it has no arguments.
|
2013-08-07 11:23:24 -06:00
|
|
|
TEXT runtime·morestack(SB),NOSPLIT,$0-0
|
2010-08-04 18:50:22 -06:00
|
|
|
// Cannot grow scheduler stack (m->g0).
|
2014-09-10 07:25:05 -06:00
|
|
|
get_tls(CX)
|
runtime: assume precisestack, copystack, StackCopyAlways, ScanStackByFrames
Commit to stack copying for stack growth.
We're carrying around a surprising amount of cruft from older schemes.
I am confident that precise stack scans and stack copying are here to stay.
Delete fallback code for when precise stack info is disabled.
Delete fallback code for when copying stacks is disabled.
Delete fallback code for when StackCopyAlways is disabled.
Delete Stktop chain - there is only one stack segment now.
Delete M.moreargp, M.moreargsize, M.moreframesize, M.cret.
Delete G.writenbuf (unrelated, just dead).
Delete runtime.lessstack, runtime.oldstack.
Delete many amd64 morestack variants.
Delete initialization of morestack frame/arg sizes (shortens split prologue!).
Replace G's stackguard/stackbase/stack0/stacksize/
syscallstack/syscallguard/forkstackguard with simple stack
bounds (lo, hi).
Update liblink, runtime/cgo for adjustments to G.
LGTM=khr
R=khr, bradfitz
CC=golang-codereviews, iant, r
https://golang.org/cl/137410043
2014-09-09 11:39:57 -06:00
|
|
|
MOVQ g(CX), BX
|
|
|
|
|
MOVQ g_m(BX), BX
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ m_g0(BX), SI
|
|
|
|
|
CMPQ g(CX), SI
|
|
|
|
|
JNE 2(PC)
|
|
|
|
|
INT $3
|
|
|
|
|
|
2014-09-05 14:51:45 -06:00
|
|
|
// Cannot grow signal stack (m->gsignal).
|
|
|
|
|
MOVQ m_gsignal(BX), SI
|
|
|
|
|
CMPQ g(CX), SI
|
|
|
|
|
JNE 2(PC)
|
|
|
|
|
INT $3
|
|
|
|
|
|
2009-06-17 16:12:16 -06:00
|
|
|
// Called from f.
|
|
|
|
|
// Set m->morebuf to f's caller.
|
|
|
|
|
MOVQ 8(SP), AX // f's caller's PC
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ AX, (m_morebuf+gobuf_pc)(BX)
|
2009-06-17 16:12:16 -06:00
|
|
|
LEAQ 16(SP), AX // f's caller's SP
|
2010-08-04 18:50:22 -06:00
|
|
|
MOVQ AX, (m_morebuf+gobuf_sp)(BX)
|
|
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ g(CX), SI
|
|
|
|
|
MOVQ SI, (m_morebuf+gobuf_g)(BX)
|
2009-06-17 16:12:16 -06:00
|
|
|
|
runtime: record proper goroutine state during stack split
Until now, the goroutine state has been scattered during the
execution of newstack and oldstack. It's all there, and those routines
know how to get back to a working goroutine, but other pieces of
the system, like stack traces, do not. If something does interrupt
the newstack or oldstack execution, the rest of the system can't
understand the goroutine. For example, if newstack decides there
is an overflow and calls throw, the stack tracer wouldn't dump the
goroutine correctly.
For newstack to save a useful state snapshot, it needs to be able
to rewind the PC in the function that triggered the split back to
the beginning of the function. (The PC is a few instructions in, just
after the call to morestack.) To make that possible, we change the
prologues to insert a jmp back to the beginning of the function
after the call to morestack. That is, the prologue used to be roughly:
TEXT myfunc
check for split
jmpcond nosplit
call morestack
nosplit:
sub $xxx, sp
Now an extra instruction is inserted after the call:
TEXT myfunc
start:
check for split
jmpcond nosplit
call morestack
jmp start
nosplit:
sub $xxx, sp
The jmp is not executed directly. It is decoded and simulated by
runtime.rewindmorestack to discover the beginning of the function,
and then the call to morestack returns directly to the start label
instead of to the jump instruction. So logically the jmp is still
executed, just not by the cpu.
The prologue thus repeats in the case of a function that needs a
stack split, but against the cost of the split itself, the extra few
instructions are noise. The repeated prologue has the nice effect of
making a stack split double-check that the new stack is big enough:
if morestack happens to return on a too-small stack, we'll now notice
before corruption happens.
The ability for newstack to rewind to the beginning of the function
should help preemption too. If newstack decides that it was called
for preemption instead of a stack split, it now has the goroutine state
correctly paused if rescheduling is needed, and when the goroutine
can run again, it can return to the start label on its original stack
and re-execute the split check.
Here is an example of a split stack overflow showing the full
trace, without any special cases in the stack printer.
(This one was triggered by making the split check incorrect.)
runtime: newstack framesize=0x0 argsize=0x18 sp=0x6aebd0 stack=[0x6b0000, 0x6b0fa0]
morebuf={pc:0x69f5b sp:0x6aebd8 lr:0x0}
sched={pc:0x68880 sp:0x6aebd0 lr:0x0 ctxt:0x34e700}
runtime: split stack overflow: 0x6aebd0 < 0x6b0000
fatal error: runtime: split stack overflow
goroutine 1 [stack split]:
runtime.mallocgc(0x290, 0x100000000, 0x1)
/Users/rsc/g/go/src/pkg/runtime/zmalloc_darwin_amd64.c:21 fp=0x6aebd8
runtime.new()
/Users/rsc/g/go/src/pkg/runtime/zmalloc_darwin_amd64.c:682 +0x5b fp=0x6aec08
go/build.(*Context).Import(0x5ae340, 0xc210030c71, 0xa, 0xc2100b4380, 0x1b, ...)
/Users/rsc/g/go/src/pkg/go/build/build.go:424 +0x3a fp=0x6b00a0
main.loadImport(0xc210030c71, 0xa, 0xc2100b4380, 0x1b, 0xc2100b42c0, ...)
/Users/rsc/g/go/src/cmd/go/pkg.go:249 +0x371 fp=0x6b01a8
main.(*Package).load(0xc21017c800, 0xc2100b42c0, 0xc2101828c0, 0x0, 0x0, ...)
/Users/rsc/g/go/src/cmd/go/pkg.go:431 +0x2801 fp=0x6b0c98
main.loadPackage(0x369040, 0x7, 0xc2100b42c0, 0x0)
/Users/rsc/g/go/src/cmd/go/pkg.go:709 +0x857 fp=0x6b0f80
----- stack segment boundary -----
main.(*builder).action(0xc2100902a0, 0x0, 0x0, 0xc2100e6c00, 0xc2100e5750, ...)
/Users/rsc/g/go/src/cmd/go/build.go:539 +0x437 fp=0x6b14a0
main.(*builder).action(0xc2100902a0, 0x0, 0x0, 0xc21015b400, 0x2, ...)
/Users/rsc/g/go/src/cmd/go/build.go:528 +0x1d2 fp=0x6b1658
main.(*builder).test(0xc2100902a0, 0xc210092000, 0x0, 0x0, 0xc21008ff60, ...)
/Users/rsc/g/go/src/cmd/go/test.go:622 +0x1b53 fp=0x6b1f68
----- stack segment boundary -----
main.runTest(0x5a6b20, 0xc21000a020, 0x2, 0x2)
/Users/rsc/g/go/src/cmd/go/test.go:366 +0xd09 fp=0x6a5cf0
main.main()
/Users/rsc/g/go/src/cmd/go/main.go:161 +0x4f9 fp=0x6a5f78
runtime.main()
/Users/rsc/g/go/src/pkg/runtime/proc.c:183 +0x92 fp=0x6a5fa0
runtime.goexit()
/Users/rsc/g/go/src/pkg/runtime/proc.c:1266 fp=0x6a5fa8
And here is a seg fault during oldstack:
SIGSEGV: segmentation violation
PC=0x1b2a6
runtime.oldstack()
/Users/rsc/g/go/src/pkg/runtime/stack.c:159 +0x76
runtime.lessstack()
/Users/rsc/g/go/src/pkg/runtime/asm_amd64.s:270 +0x22
goroutine 1 [stack unsplit]:
fmt.(*pp).printArg(0x2102e64e0, 0xe5c80, 0x2102c9220, 0x73, 0x0, ...)
/Users/rsc/g/go/src/pkg/fmt/print.go:818 +0x3d3 fp=0x221031e6f8
fmt.(*pp).doPrintf(0x2102e64e0, 0x12fb20, 0x2, 0x221031eb98, 0x1, ...)
/Users/rsc/g/go/src/pkg/fmt/print.go:1183 +0x15cb fp=0x221031eaf0
fmt.Sprintf(0x12fb20, 0x2, 0x221031eb98, 0x1, 0x1, ...)
/Users/rsc/g/go/src/pkg/fmt/print.go:234 +0x67 fp=0x221031eb40
flag.(*stringValue).String(0x2102c9210, 0x1, 0x0)
/Users/rsc/g/go/src/pkg/flag/flag.go:180 +0xb3 fp=0x221031ebb0
flag.(*FlagSet).Var(0x2102f6000, 0x293d38, 0x2102c9210, 0x143490, 0xa, ...)
/Users/rsc/g/go/src/pkg/flag/flag.go:633 +0x40 fp=0x221031eca0
flag.(*FlagSet).StringVar(0x2102f6000, 0x2102c9210, 0x143490, 0xa, 0x12fa60, ...)
/Users/rsc/g/go/src/pkg/flag/flag.go:550 +0x91 fp=0x221031ece8
flag.(*FlagSet).String(0x2102f6000, 0x143490, 0xa, 0x12fa60, 0x0, ...)
/Users/rsc/g/go/src/pkg/flag/flag.go:563 +0x87 fp=0x221031ed38
flag.String(0x143490, 0xa, 0x12fa60, 0x0, 0x161950, ...)
/Users/rsc/g/go/src/pkg/flag/flag.go:570 +0x6b fp=0x221031ed80
testing.init()
/Users/rsc/g/go/src/pkg/testing/testing.go:-531 +0xbb fp=0x221031edc0
strings_test.init()
/Users/rsc/g/go/src/pkg/strings/strings_test.go:1115 +0x62 fp=0x221031ef70
main.init()
strings/_test/_testmain.go:90 +0x3d fp=0x221031ef78
runtime.main()
/Users/rsc/g/go/src/pkg/runtime/proc.c:180 +0x8a fp=0x221031efa0
runtime.goexit()
/Users/rsc/g/go/src/pkg/runtime/proc.c:1269 fp=0x221031efa8
goroutine 2 [runnable]:
runtime.MHeap_Scavenger()
/Users/rsc/g/go/src/pkg/runtime/mheap.c:438
runtime.goexit()
/Users/rsc/g/go/src/pkg/runtime/proc.c:1269
created by runtime.main
/Users/rsc/g/go/src/pkg/runtime/proc.c:166
rax 0x23ccc0
rbx 0x23ccc0
rcx 0x0
rdx 0x38
rdi 0x2102c0170
rsi 0x221032cfe0
rbp 0x221032cfa0
rsp 0x7fff5fbff5b0
r8 0x2102c0120
r9 0x221032cfa0
r10 0x221032c000
r11 0x104ce8
r12 0xe5c80
r13 0x1be82baac718
r14 0x13091135f7d69200
r15 0x0
rip 0x1b2a6
rflags 0x10246
cs 0x2b
fs 0x0
gs 0x0
Fixes #5723.
R=r, dvyukov, go.peter.90, dave, iant
CC=golang-dev
https://golang.org/cl/10360048
2013-06-27 09:32:01 -06:00
|
|
|
// Set g->sched to context in f.
|
|
|
|
|
MOVQ 0(SP), AX // f's PC
|
|
|
|
|
MOVQ AX, (g_sched+gobuf_pc)(SI)
|
|
|
|
|
MOVQ SI, (g_sched+gobuf_g)(SI)
|
|
|
|
|
LEAQ 8(SP), AX // f's SP
|
|
|
|
|
MOVQ AX, (g_sched+gobuf_sp)(SI)
|
|
|
|
|
MOVQ DX, (g_sched+gobuf_ctxt)(SI)
|
2015-01-14 09:09:50 -07:00
|
|
|
MOVQ BP, (g_sched+gobuf_bp)(SI)
|
2009-06-17 16:12:16 -06:00
|
|
|
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// Call newstack on m->g0's stack.
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ m_g0(BX), BX
|
|
|
|
|
MOVQ BX, g(CX)
|
|
|
|
|
MOVQ (g_sched+gobuf_sp)(BX), SP
|
runtime: ,s/[a-zA-Z0-9_]+/runtime·&/g, almost
Prefix all external symbols in runtime by runtime·,
to avoid conflicts with possible symbols of the same
name in linked-in C libraries. The obvious conflicts
are printf, malloc, and free, but hide everything to
avoid future pain.
The symbols left alone are:
** known to cgo **
_cgo_free
_cgo_malloc
libcgo_thread_start
initcgo
ncgocall
** known to linker **
_rt0_$GOARCH
_rt0_$GOARCH_$GOOS
text
etext
data
end
pclntab
epclntab
symtab
esymtab
** known to C compiler **
_divv
_modv
_div64by32
etc (arch specific)
Tested on darwin/386, darwin/amd64, linux/386, linux/amd64.
Built (but not tested) for freebsd/386, freebsd/amd64, linux/arm, windows/386.
R=r, PeterGo
CC=golang-dev
https://golang.org/cl/2899041
2010-11-04 12:00:19 -06:00
|
|
|
CALL runtime·newstack(SB)
|
2009-06-17 16:12:16 -06:00
|
|
|
MOVQ $0, 0x1003 // crash if newstack returns
|
2009-07-08 19:16:09 -06:00
|
|
|
RET
|
|
|
|
|
|
runtime: assume precisestack, copystack, StackCopyAlways, ScanStackByFrames
Commit to stack copying for stack growth.
We're carrying around a surprising amount of cruft from older schemes.
I am confident that precise stack scans and stack copying are here to stay.
Delete fallback code for when precise stack info is disabled.
Delete fallback code for when copying stacks is disabled.
Delete fallback code for when StackCopyAlways is disabled.
Delete Stktop chain - there is only one stack segment now.
Delete M.moreargp, M.moreargsize, M.moreframesize, M.cret.
Delete G.writenbuf (unrelated, just dead).
Delete runtime.lessstack, runtime.oldstack.
Delete many amd64 morestack variants.
Delete initialization of morestack frame/arg sizes (shortens split prologue!).
Replace G's stackguard/stackbase/stack0/stacksize/
syscallstack/syscallguard/forkstackguard with simple stack
bounds (lo, hi).
Update liblink, runtime/cgo for adjustments to G.
LGTM=khr
R=khr, bradfitz
CC=golang-codereviews, iant, r
https://golang.org/cl/137410043
2014-09-09 11:39:57 -06:00
|
|
|
// morestack but not preserving ctxt.
|
|
|
|
|
TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
|
|
|
|
|
MOVL $0, DX
|
|
|
|
|
JMP runtime·morestack(SB)
|
|
|
|
|
|
runtime: implement GC stack barriers
This commit implements stack barriers to minimize the amount of
stack re-scanning that must be done during mark termination.
Currently the GC scans stacks of active goroutines twice during every
GC cycle: once at the beginning during root discovery and once at the
end during mark termination. The second scan happens while the world
is stopped and guarantees that we've seen all of the roots (since
there are no write barriers on writes to local stack
variables). However, this means pause time is proportional to stack
size. In particularly recursive programs, this can drive pause time up
past our 10ms goal (e.g., it takes about 150ms to scan a 50MB heap).
Re-scanning the entire stack is rarely necessary, especially for large
stacks, because usually most of the frames on the stack were not
active between the first and second scans and hence any changes to
these frames (via non-escaping pointers passed down the stack) were
tracked by write barriers.
To efficiently track how far a stack has been unwound since the first
scan (and, hence, how much needs to be re-scanned), this commit
introduces stack barriers. During the first scan, at exponentially
spaced points in each stack, the scan overwrites return PCs with the
PC of the stack barrier function. When "returned" to, the stack
barrier function records how far the stack has unwound and jumps to
the original return PC for that point in the stack. Then the second
scan only needs to proceed as far as the lowest barrier that hasn't
been hit.
For deeply recursive programs, this substantially reduces mark
termination time (and hence pause time). For the goscheme example
linked in issue #10898, prior to this change, mark termination times
were typically between 100 and 500ms; with this change, mark
termination times are typically between 10 and 20ms. As a result of
the reduced stack scanning work, this reduces overall execution time
of the goscheme example by 20%.
Fixes #10898.
The effect of this on programs that are not deeply recursive is
minimal:
name old time/op new time/op delta
BinaryTree17 3.16s ± 2% 3.26s ± 1% +3.31% (p=0.000 n=19+19)
Fannkuch11 2.42s ± 1% 2.48s ± 1% +2.24% (p=0.000 n=17+19)
FmtFprintfEmpty 50.0ns ± 3% 49.8ns ± 1% ~ (p=0.534 n=20+19)
FmtFprintfString 173ns ± 0% 175ns ± 0% +1.49% (p=0.000 n=16+19)
FmtFprintfInt 170ns ± 1% 175ns ± 1% +2.97% (p=0.000 n=20+19)
FmtFprintfIntInt 288ns ± 0% 295ns ± 0% +2.73% (p=0.000 n=16+19)
FmtFprintfPrefixedInt 242ns ± 1% 252ns ± 1% +4.13% (p=0.000 n=18+18)
FmtFprintfFloat 324ns ± 0% 323ns ± 0% -0.36% (p=0.000 n=20+19)
FmtManyArgs 1.14µs ± 0% 1.12µs ± 1% -1.01% (p=0.000 n=18+19)
GobDecode 8.88ms ± 1% 8.87ms ± 0% ~ (p=0.480 n=19+18)
GobEncode 6.80ms ± 1% 6.85ms ± 0% +0.82% (p=0.000 n=20+18)
Gzip 363ms ± 1% 363ms ± 1% ~ (p=0.077 n=18+20)
Gunzip 90.6ms ± 0% 90.0ms ± 1% -0.71% (p=0.000 n=17+18)
HTTPClientServer 51.5µs ± 1% 50.8µs ± 1% -1.32% (p=0.000 n=18+18)
JSONEncode 17.0ms ± 0% 17.1ms ± 0% +0.40% (p=0.000 n=18+17)
JSONDecode 61.8ms ± 0% 63.8ms ± 1% +3.11% (p=0.000 n=18+17)
Mandelbrot200 3.84ms ± 0% 3.84ms ± 1% ~ (p=0.583 n=19+19)
GoParse 3.71ms ± 1% 3.72ms ± 1% ~ (p=0.159 n=18+19)
RegexpMatchEasy0_32 100ns ± 0% 100ns ± 1% -0.19% (p=0.033 n=17+19)
RegexpMatchEasy0_1K 342ns ± 1% 331ns ± 0% -3.41% (p=0.000 n=19+19)
RegexpMatchEasy1_32 82.5ns ± 0% 81.7ns ± 0% -0.98% (p=0.000 n=18+18)
RegexpMatchEasy1_1K 505ns ± 0% 494ns ± 1% -2.16% (p=0.000 n=18+18)
RegexpMatchMedium_32 137ns ± 1% 137ns ± 1% -0.24% (p=0.048 n=20+18)
RegexpMatchMedium_1K 41.6µs ± 0% 41.3µs ± 1% -0.57% (p=0.004 n=18+20)
RegexpMatchHard_32 2.11µs ± 0% 2.11µs ± 1% +0.20% (p=0.037 n=17+19)
RegexpMatchHard_1K 63.9µs ± 2% 63.3µs ± 0% -0.99% (p=0.000 n=20+17)
Revcomp 560ms ± 1% 522ms ± 0% -6.87% (p=0.000 n=18+16)
Template 75.0ms ± 0% 75.1ms ± 1% +0.18% (p=0.013 n=18+19)
TimeParse 358ns ± 1% 364ns ± 0% +1.74% (p=0.000 n=20+15)
TimeFormat 360ns ± 0% 372ns ± 0% +3.55% (p=0.000 n=20+18)
Change-Id: If8a9bfae6c128d15a4f405e02bcfa50129df82a2
Reviewed-on: https://go-review.googlesource.com/10314
Reviewed-by: Russ Cox <rsc@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
2015-05-20 14:30:49 -06:00
|
|
|
TEXT runtime·stackBarrier(SB),NOSPLIT,$0
|
|
|
|
|
// We came here via a RET to an overwritten return PC.
|
|
|
|
|
// AX may be live. Other registers are available.
|
|
|
|
|
|
|
|
|
|
// Get the original return PC, g.stkbar[g.stkbarPos].savedLRVal.
|
|
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ g(CX), CX
|
|
|
|
|
MOVQ (g_stkbar+slice_array)(CX), DX
|
|
|
|
|
MOVQ g_stkbarPos(CX), BX
|
|
|
|
|
IMULQ $stkbar__size, BX // Too big for SIB.
|
2015-08-26 13:06:43 -06:00
|
|
|
MOVQ stkbar_savedLRPtr(DX)(BX*1), R8
|
runtime: implement GC stack barriers
This commit implements stack barriers to minimize the amount of
stack re-scanning that must be done during mark termination.
Currently the GC scans stacks of active goroutines twice during every
GC cycle: once at the beginning during root discovery and once at the
end during mark termination. The second scan happens while the world
is stopped and guarantees that we've seen all of the roots (since
there are no write barriers on writes to local stack
variables). However, this means pause time is proportional to stack
size. In particularly recursive programs, this can drive pause time up
past our 10ms goal (e.g., it takes about 150ms to scan a 50MB heap).
Re-scanning the entire stack is rarely necessary, especially for large
stacks, because usually most of the frames on the stack were not
active between the first and second scans and hence any changes to
these frames (via non-escaping pointers passed down the stack) were
tracked by write barriers.
To efficiently track how far a stack has been unwound since the first
scan (and, hence, how much needs to be re-scanned), this commit
introduces stack barriers. During the first scan, at exponentially
spaced points in each stack, the scan overwrites return PCs with the
PC of the stack barrier function. When "returned" to, the stack
barrier function records how far the stack has unwound and jumps to
the original return PC for that point in the stack. Then the second
scan only needs to proceed as far as the lowest barrier that hasn't
been hit.
For deeply recursive programs, this substantially reduces mark
termination time (and hence pause time). For the goscheme example
linked in issue #10898, prior to this change, mark termination times
were typically between 100 and 500ms; with this change, mark
termination times are typically between 10 and 20ms. As a result of
the reduced stack scanning work, this reduces overall execution time
of the goscheme example by 20%.
Fixes #10898.
The effect of this on programs that are not deeply recursive is
minimal:
name old time/op new time/op delta
BinaryTree17 3.16s ± 2% 3.26s ± 1% +3.31% (p=0.000 n=19+19)
Fannkuch11 2.42s ± 1% 2.48s ± 1% +2.24% (p=0.000 n=17+19)
FmtFprintfEmpty 50.0ns ± 3% 49.8ns ± 1% ~ (p=0.534 n=20+19)
FmtFprintfString 173ns ± 0% 175ns ± 0% +1.49% (p=0.000 n=16+19)
FmtFprintfInt 170ns ± 1% 175ns ± 1% +2.97% (p=0.000 n=20+19)
FmtFprintfIntInt 288ns ± 0% 295ns ± 0% +2.73% (p=0.000 n=16+19)
FmtFprintfPrefixedInt 242ns ± 1% 252ns ± 1% +4.13% (p=0.000 n=18+18)
FmtFprintfFloat 324ns ± 0% 323ns ± 0% -0.36% (p=0.000 n=20+19)
FmtManyArgs 1.14µs ± 0% 1.12µs ± 1% -1.01% (p=0.000 n=18+19)
GobDecode 8.88ms ± 1% 8.87ms ± 0% ~ (p=0.480 n=19+18)
GobEncode 6.80ms ± 1% 6.85ms ± 0% +0.82% (p=0.000 n=20+18)
Gzip 363ms ± 1% 363ms ± 1% ~ (p=0.077 n=18+20)
Gunzip 90.6ms ± 0% 90.0ms ± 1% -0.71% (p=0.000 n=17+18)
HTTPClientServer 51.5µs ± 1% 50.8µs ± 1% -1.32% (p=0.000 n=18+18)
JSONEncode 17.0ms ± 0% 17.1ms ± 0% +0.40% (p=0.000 n=18+17)
JSONDecode 61.8ms ± 0% 63.8ms ± 1% +3.11% (p=0.000 n=18+17)
Mandelbrot200 3.84ms ± 0% 3.84ms ± 1% ~ (p=0.583 n=19+19)
GoParse 3.71ms ± 1% 3.72ms ± 1% ~ (p=0.159 n=18+19)
RegexpMatchEasy0_32 100ns ± 0% 100ns ± 1% -0.19% (p=0.033 n=17+19)
RegexpMatchEasy0_1K 342ns ± 1% 331ns ± 0% -3.41% (p=0.000 n=19+19)
RegexpMatchEasy1_32 82.5ns ± 0% 81.7ns ± 0% -0.98% (p=0.000 n=18+18)
RegexpMatchEasy1_1K 505ns ± 0% 494ns ± 1% -2.16% (p=0.000 n=18+18)
RegexpMatchMedium_32 137ns ± 1% 137ns ± 1% -0.24% (p=0.048 n=20+18)
RegexpMatchMedium_1K 41.6µs ± 0% 41.3µs ± 1% -0.57% (p=0.004 n=18+20)
RegexpMatchHard_32 2.11µs ± 0% 2.11µs ± 1% +0.20% (p=0.037 n=17+19)
RegexpMatchHard_1K 63.9µs ± 2% 63.3µs ± 0% -0.99% (p=0.000 n=20+17)
Revcomp 560ms ± 1% 522ms ± 0% -6.87% (p=0.000 n=18+16)
Template 75.0ms ± 0% 75.1ms ± 1% +0.18% (p=0.013 n=18+19)
TimeParse 358ns ± 1% 364ns ± 0% +1.74% (p=0.000 n=20+15)
TimeFormat 360ns ± 0% 372ns ± 0% +3.55% (p=0.000 n=20+18)
Change-Id: If8a9bfae6c128d15a4f405e02bcfa50129df82a2
Reviewed-on: https://go-review.googlesource.com/10314
Reviewed-by: Russ Cox <rsc@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
2015-05-20 14:30:49 -06:00
|
|
|
MOVQ stkbar_savedLRVal(DX)(BX*1), BX
|
2015-08-26 13:06:43 -06:00
|
|
|
// Assert that we're popping the right saved LR.
|
2015-11-18 14:48:22 -07:00
|
|
|
ADDQ $8, R8
|
2015-08-26 13:06:43 -06:00
|
|
|
CMPQ R8, SP
|
2015-11-18 14:48:22 -07:00
|
|
|
JEQ 2(PC)
|
2015-08-26 13:06:43 -06:00
|
|
|
MOVL $0, 0
|
runtime: implement GC stack barriers
This commit implements stack barriers to minimize the amount of
stack re-scanning that must be done during mark termination.
Currently the GC scans stacks of active goroutines twice during every
GC cycle: once at the beginning during root discovery and once at the
end during mark termination. The second scan happens while the world
is stopped and guarantees that we've seen all of the roots (since
there are no write barriers on writes to local stack
variables). However, this means pause time is proportional to stack
size. In particularly recursive programs, this can drive pause time up
past our 10ms goal (e.g., it takes about 150ms to scan a 50MB heap).
Re-scanning the entire stack is rarely necessary, especially for large
stacks, because usually most of the frames on the stack were not
active between the first and second scans and hence any changes to
these frames (via non-escaping pointers passed down the stack) were
tracked by write barriers.
To efficiently track how far a stack has been unwound since the first
scan (and, hence, how much needs to be re-scanned), this commit
introduces stack barriers. During the first scan, at exponentially
spaced points in each stack, the scan overwrites return PCs with the
PC of the stack barrier function. When "returned" to, the stack
barrier function records how far the stack has unwound and jumps to
the original return PC for that point in the stack. Then the second
scan only needs to proceed as far as the lowest barrier that hasn't
been hit.
For deeply recursive programs, this substantially reduces mark
termination time (and hence pause time). For the goscheme example
linked in issue #10898, prior to this change, mark termination times
were typically between 100 and 500ms; with this change, mark
termination times are typically between 10 and 20ms. As a result of
the reduced stack scanning work, this reduces overall execution time
of the goscheme example by 20%.
Fixes #10898.
The effect of this on programs that are not deeply recursive is
minimal:
name old time/op new time/op delta
BinaryTree17 3.16s ± 2% 3.26s ± 1% +3.31% (p=0.000 n=19+19)
Fannkuch11 2.42s ± 1% 2.48s ± 1% +2.24% (p=0.000 n=17+19)
FmtFprintfEmpty 50.0ns ± 3% 49.8ns ± 1% ~ (p=0.534 n=20+19)
FmtFprintfString 173ns ± 0% 175ns ± 0% +1.49% (p=0.000 n=16+19)
FmtFprintfInt 170ns ± 1% 175ns ± 1% +2.97% (p=0.000 n=20+19)
FmtFprintfIntInt 288ns ± 0% 295ns ± 0% +2.73% (p=0.000 n=16+19)
FmtFprintfPrefixedInt 242ns ± 1% 252ns ± 1% +4.13% (p=0.000 n=18+18)
FmtFprintfFloat 324ns ± 0% 323ns ± 0% -0.36% (p=0.000 n=20+19)
FmtManyArgs 1.14µs ± 0% 1.12µs ± 1% -1.01% (p=0.000 n=18+19)
GobDecode 8.88ms ± 1% 8.87ms ± 0% ~ (p=0.480 n=19+18)
GobEncode 6.80ms ± 1% 6.85ms ± 0% +0.82% (p=0.000 n=20+18)
Gzip 363ms ± 1% 363ms ± 1% ~ (p=0.077 n=18+20)
Gunzip 90.6ms ± 0% 90.0ms ± 1% -0.71% (p=0.000 n=17+18)
HTTPClientServer 51.5µs ± 1% 50.8µs ± 1% -1.32% (p=0.000 n=18+18)
JSONEncode 17.0ms ± 0% 17.1ms ± 0% +0.40% (p=0.000 n=18+17)
JSONDecode 61.8ms ± 0% 63.8ms ± 1% +3.11% (p=0.000 n=18+17)
Mandelbrot200 3.84ms ± 0% 3.84ms ± 1% ~ (p=0.583 n=19+19)
GoParse 3.71ms ± 1% 3.72ms ± 1% ~ (p=0.159 n=18+19)
RegexpMatchEasy0_32 100ns ± 0% 100ns ± 1% -0.19% (p=0.033 n=17+19)
RegexpMatchEasy0_1K 342ns ± 1% 331ns ± 0% -3.41% (p=0.000 n=19+19)
RegexpMatchEasy1_32 82.5ns ± 0% 81.7ns ± 0% -0.98% (p=0.000 n=18+18)
RegexpMatchEasy1_1K 505ns ± 0% 494ns ± 1% -2.16% (p=0.000 n=18+18)
RegexpMatchMedium_32 137ns ± 1% 137ns ± 1% -0.24% (p=0.048 n=20+18)
RegexpMatchMedium_1K 41.6µs ± 0% 41.3µs ± 1% -0.57% (p=0.004 n=18+20)
RegexpMatchHard_32 2.11µs ± 0% 2.11µs ± 1% +0.20% (p=0.037 n=17+19)
RegexpMatchHard_1K 63.9µs ± 2% 63.3µs ± 0% -0.99% (p=0.000 n=20+17)
Revcomp 560ms ± 1% 522ms ± 0% -6.87% (p=0.000 n=18+16)
Template 75.0ms ± 0% 75.1ms ± 1% +0.18% (p=0.013 n=18+19)
TimeParse 358ns ± 1% 364ns ± 0% +1.74% (p=0.000 n=20+15)
TimeFormat 360ns ± 0% 372ns ± 0% +3.55% (p=0.000 n=20+18)
Change-Id: If8a9bfae6c128d15a4f405e02bcfa50129df82a2
Reviewed-on: https://go-review.googlesource.com/10314
Reviewed-by: Russ Cox <rsc@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
2015-05-20 14:30:49 -06:00
|
|
|
// Record that this stack barrier was hit.
|
|
|
|
|
ADDQ $1, g_stkbarPos(CX)
|
|
|
|
|
// Jump to the original return PC.
|
|
|
|
|
JMP BX
|
|
|
|
|
|
2014-09-08 11:14:41 -06:00
|
|
|
// reflectcall: call a function with the given argument list
|
2014-12-30 11:59:55 -07:00
|
|
|
// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
|
2013-08-02 14:03:14 -06:00
|
|
|
// we don't have variable-sized frames, so we use a small number
|
|
|
|
|
// of constant-sized-frame functions to encode a few bits of size in the pc.
|
|
|
|
|
// Caution: ugly multiline assembly macros in your future!
|
|
|
|
|
|
|
|
|
|
#define DISPATCH(NAME,MAXSIZE) \
|
|
|
|
|
CMPQ CX, $MAXSIZE; \
|
|
|
|
|
JA 3(PC); \
|
2014-10-15 11:12:16 -06:00
|
|
|
MOVQ $NAME(SB), AX; \
|
2013-08-02 14:03:14 -06:00
|
|
|
JMP AX
|
2014-07-30 11:11:44 -06:00
|
|
|
// Note: can't just "JMP NAME(SB)" - bad inlining results.
|
2013-08-02 14:03:14 -06:00
|
|
|
|
2014-12-22 11:27:53 -07:00
|
|
|
TEXT reflect·call(SB), NOSPLIT, $0-0
|
|
|
|
|
JMP ·reflectcall(SB)
|
|
|
|
|
|
2014-12-30 11:59:55 -07:00
|
|
|
TEXT ·reflectcall(SB), NOSPLIT, $0-32
|
|
|
|
|
MOVLQZX argsize+24(FP), CX
|
|
|
|
|
// NOTE(rsc): No call16, because CALLFN needs four words
|
|
|
|
|
// of argument space to invoke callwritebarrier.
|
2014-07-30 11:11:44 -06:00
|
|
|
DISPATCH(runtime·call32, 32)
|
|
|
|
|
DISPATCH(runtime·call64, 64)
|
|
|
|
|
DISPATCH(runtime·call128, 128)
|
|
|
|
|
DISPATCH(runtime·call256, 256)
|
|
|
|
|
DISPATCH(runtime·call512, 512)
|
|
|
|
|
DISPATCH(runtime·call1024, 1024)
|
|
|
|
|
DISPATCH(runtime·call2048, 2048)
|
|
|
|
|
DISPATCH(runtime·call4096, 4096)
|
|
|
|
|
DISPATCH(runtime·call8192, 8192)
|
|
|
|
|
DISPATCH(runtime·call16384, 16384)
|
|
|
|
|
DISPATCH(runtime·call32768, 32768)
|
|
|
|
|
DISPATCH(runtime·call65536, 65536)
|
|
|
|
|
DISPATCH(runtime·call131072, 131072)
|
|
|
|
|
DISPATCH(runtime·call262144, 262144)
|
|
|
|
|
DISPATCH(runtime·call524288, 524288)
|
|
|
|
|
DISPATCH(runtime·call1048576, 1048576)
|
|
|
|
|
DISPATCH(runtime·call2097152, 2097152)
|
|
|
|
|
DISPATCH(runtime·call4194304, 4194304)
|
|
|
|
|
DISPATCH(runtime·call8388608, 8388608)
|
|
|
|
|
DISPATCH(runtime·call16777216, 16777216)
|
|
|
|
|
DISPATCH(runtime·call33554432, 33554432)
|
|
|
|
|
DISPATCH(runtime·call67108864, 67108864)
|
|
|
|
|
DISPATCH(runtime·call134217728, 134217728)
|
|
|
|
|
DISPATCH(runtime·call268435456, 268435456)
|
|
|
|
|
DISPATCH(runtime·call536870912, 536870912)
|
|
|
|
|
DISPATCH(runtime·call1073741824, 1073741824)
|
2013-08-02 14:03:14 -06:00
|
|
|
MOVQ $runtime·badreflectcall(SB), AX
|
|
|
|
|
JMP AX
|
|
|
|
|
|
2013-08-06 15:33:55 -06:00
|
|
|
#define CALLFN(NAME,MAXSIZE) \
|
2014-12-30 11:59:55 -07:00
|
|
|
TEXT NAME(SB), WRAPPER, $MAXSIZE-32; \
|
2014-10-15 11:12:16 -06:00
|
|
|
NO_LOCAL_POINTERS; \
|
2013-08-02 14:03:14 -06:00
|
|
|
/* copy arguments to stack */ \
|
2014-12-30 11:59:55 -07:00
|
|
|
MOVQ argptr+16(FP), SI; \
|
|
|
|
|
MOVLQZX argsize+24(FP), CX; \
|
2013-08-02 14:03:14 -06:00
|
|
|
MOVQ SP, DI; \
|
|
|
|
|
REP;MOVSB; \
|
|
|
|
|
/* call function */ \
|
2014-12-30 11:59:55 -07:00
|
|
|
MOVQ f+8(FP), DX; \
|
2014-05-21 15:28:34 -06:00
|
|
|
PCDATA $PCDATA_StackMapIndex, $0; \
|
2013-08-02 14:03:14 -06:00
|
|
|
CALL (DX); \
|
|
|
|
|
/* copy return values back */ \
|
2014-12-30 11:59:55 -07:00
|
|
|
MOVQ argptr+16(FP), DI; \
|
|
|
|
|
MOVLQZX argsize+24(FP), CX; \
|
|
|
|
|
MOVLQZX retoffset+28(FP), BX; \
|
2013-08-02 14:03:14 -06:00
|
|
|
MOVQ SP, SI; \
|
reflect, runtime: fix crash in GC due to reflect.call + precise GC
Given
type Outer struct {
*Inner
...
}
the compiler generates the implementation of (*Outer).M dispatching to
the embedded Inner. The implementation is logically:
func (p *Outer) M() {
(p.Inner).M()
}
but since the only change here is the replacement of one pointer
receiver with another, the actual generated code overwrites the
original receiver with the p.Inner pointer and then jumps to the M
method expecting the *Inner receiver.
During reflect.Value.Call, we create an argument frame and the
associated data structures to describe it to the garbage collector,
populate the frame, call reflect.call to run a function call using
that frame, and then copy the results back out of the frame. The
reflect.call function does a memmove of the frame structure onto the
stack (to set up the inputs), runs the call, and the memmoves the
stack back to the frame structure (to preserve the outputs).
Originally reflect.call did not distinguish inputs from outputs: both
memmoves were for the full stack frame. However, in the case where the
called function was one of these wrappers, the rewritten receiver is
almost certainly a different type than the original receiver. This is
not a problem on the stack, where we use the program counter to
determine the type information and understand that during (*Outer).M
the receiver is an *Outer while during (*Inner).M the receiver in the
same memory word is now an *Inner. But in the statically typed
argument frame created by reflect, the receiver is always an *Outer.
Copying the modified receiver pointer off the stack into the frame
will store an *Inner there, and then if a garbage collection happens
to scan that argument frame before it is discarded, it will scan the
*Inner memory as if it were an *Outer. If the two have different
memory layouts, the collection will intepret the memory incorrectly.
Fix by only copying back the results.
Fixes #7725.
LGTM=khr
R=khr
CC=dave, golang-codereviews
https://golang.org/cl/85180043
2014-04-08 09:11:35 -06:00
|
|
|
ADDQ BX, DI; \
|
|
|
|
|
ADDQ BX, SI; \
|
|
|
|
|
SUBQ BX, CX; \
|
2013-08-02 14:03:14 -06:00
|
|
|
REP;MOVSB; \
|
2014-12-30 11:59:55 -07:00
|
|
|
/* execute write barrier updates */ \
|
|
|
|
|
MOVQ argtype+0(FP), DX; \
|
|
|
|
|
MOVQ argptr+16(FP), DI; \
|
|
|
|
|
MOVLQZX argsize+24(FP), CX; \
|
|
|
|
|
MOVLQZX retoffset+28(FP), BX; \
|
|
|
|
|
MOVQ DX, 0(SP); \
|
|
|
|
|
MOVQ DI, 8(SP); \
|
|
|
|
|
MOVQ CX, 16(SP); \
|
|
|
|
|
MOVQ BX, 24(SP); \
|
|
|
|
|
CALL runtime·callwritebarrier(SB); \
|
2013-08-02 14:03:14 -06:00
|
|
|
RET
|
|
|
|
|
|
2014-10-15 11:12:16 -06:00
|
|
|
CALLFN(·call32, 32)
|
|
|
|
|
CALLFN(·call64, 64)
|
|
|
|
|
CALLFN(·call128, 128)
|
|
|
|
|
CALLFN(·call256, 256)
|
|
|
|
|
CALLFN(·call512, 512)
|
|
|
|
|
CALLFN(·call1024, 1024)
|
|
|
|
|
CALLFN(·call2048, 2048)
|
|
|
|
|
CALLFN(·call4096, 4096)
|
|
|
|
|
CALLFN(·call8192, 8192)
|
|
|
|
|
CALLFN(·call16384, 16384)
|
|
|
|
|
CALLFN(·call32768, 32768)
|
|
|
|
|
CALLFN(·call65536, 65536)
|
|
|
|
|
CALLFN(·call131072, 131072)
|
|
|
|
|
CALLFN(·call262144, 262144)
|
|
|
|
|
CALLFN(·call524288, 524288)
|
|
|
|
|
CALLFN(·call1048576, 1048576)
|
|
|
|
|
CALLFN(·call2097152, 2097152)
|
|
|
|
|
CALLFN(·call4194304, 4194304)
|
|
|
|
|
CALLFN(·call8388608, 8388608)
|
|
|
|
|
CALLFN(·call16777216, 16777216)
|
|
|
|
|
CALLFN(·call33554432, 33554432)
|
|
|
|
|
CALLFN(·call67108864, 67108864)
|
|
|
|
|
CALLFN(·call134217728, 134217728)
|
|
|
|
|
CALLFN(·call268435456, 268435456)
|
|
|
|
|
CALLFN(·call536870912, 536870912)
|
|
|
|
|
CALLFN(·call1073741824, 1073741824)
|
2013-08-02 14:03:14 -06:00
|
|
|
|
2013-08-07 11:23:24 -06:00
|
|
|
TEXT runtime·procyield(SB),NOSPLIT,$0-0
|
cmd/cc, runtime: convert C compilers to use Go calling convention
To date, the C compilers and Go compilers differed only in how
values were returned from functions. This made it difficult to call
Go from C or C from Go if return values were involved. It also made
assembly called from Go and assembly called from C different.
This CL changes the C compiler to use the Go conventions, passing
results on the stack, after the arguments.
[Exception: this does not apply to C ... functions, because you can't
know where on the stack the arguments end.]
By doing this, the CL makes it possible to rewrite C functions into Go
one at a time, without worrying about which languages call that
function or which languages it calls.
This CL also updates all the assembly files in package runtime to use
the new conventions. Argument references of the form 40(SP) have
been rewritten to the form name+10(FP) instead, and there are now
Go func prototypes for every assembly function called from C or Go.
This means that 'go vet runtime' checks effectively every assembly
function, and go vet's output was used to automate the bulk of the
conversion.
Some functions, like seek and nsec on Plan 9, needed to be rewritten.
Many assembly routines called from C were reading arguments
incorrectly, using MOVL instead of MOVQ or vice versa, especially on
the less used systems like openbsd.
These were found by go vet and have been corrected too.
If we're lucky, this may reduce flakiness on those systems.
Tested on:
darwin/386
darwin/amd64
linux/arm
linux/386
linux/amd64
If this breaks another system, the bug is almost certainly in the
sys_$GOOS_$GOARCH.s file, since the rest of the CL is tested
by the combination of the above systems.
LGTM=dvyukov, iant
R=golang-codereviews, 0intro, dave, alex.brainman, dvyukov, iant
CC=golang-codereviews, josharian, r
https://golang.org/cl/135830043
2014-08-27 09:32:17 -06:00
|
|
|
MOVL cycles+0(FP), AX
|
runtime: improve Linux mutex
The implementation is hybrid active/passive spin/blocking mutex.
The design minimizes amount of context switches and futex calls.
The idea is that all critical sections in runtime are intentially
small, so pure blocking mutex behaves badly causing
a lot of context switches, thread parking/unparking and kernel calls.
Note that some synthetic benchmarks become somewhat slower,
that's due to increased contention on other data structures,
it should not affect programs that do any real work.
On 2 x Intel E5620, 8 HT cores, 2.4GHz
benchmark old ns/op new ns/op delta
BenchmarkSelectContended 521.00 503.00 -3.45%
BenchmarkSelectContended-2 661.00 320.00 -51.59%
BenchmarkSelectContended-4 1139.00 629.00 -44.78%
BenchmarkSelectContended-8 2870.00 878.00 -69.41%
BenchmarkSelectContended-16 5276.00 818.00 -84.50%
BenchmarkChanContended 112.00 103.00 -8.04%
BenchmarkChanContended-2 631.00 174.00 -72.42%
BenchmarkChanContended-4 682.00 272.00 -60.12%
BenchmarkChanContended-8 1601.00 520.00 -67.52%
BenchmarkChanContended-16 3100.00 372.00 -88.00%
BenchmarkChanSync 253.00 239.00 -5.53%
BenchmarkChanSync-2 5030.00 4648.00 -7.59%
BenchmarkChanSync-4 4826.00 4694.00 -2.74%
BenchmarkChanSync-8 4778.00 4713.00 -1.36%
BenchmarkChanSync-16 5289.00 4710.00 -10.95%
BenchmarkChanProdCons0 273.00 254.00 -6.96%
BenchmarkChanProdCons0-2 599.00 400.00 -33.22%
BenchmarkChanProdCons0-4 1168.00 659.00 -43.58%
BenchmarkChanProdCons0-8 2831.00 1057.00 -62.66%
BenchmarkChanProdCons0-16 4197.00 1037.00 -75.29%
BenchmarkChanProdCons10 150.00 140.00 -6.67%
BenchmarkChanProdCons10-2 607.00 268.00 -55.85%
BenchmarkChanProdCons10-4 1137.00 404.00 -64.47%
BenchmarkChanProdCons10-8 2115.00 828.00 -60.85%
BenchmarkChanProdCons10-16 4283.00 855.00 -80.04%
BenchmarkChanProdCons100 117.00 110.00 -5.98%
BenchmarkChanProdCons100-2 558.00 218.00 -60.93%
BenchmarkChanProdCons100-4 722.00 287.00 -60.25%
BenchmarkChanProdCons100-8 1840.00 431.00 -76.58%
BenchmarkChanProdCons100-16 3394.00 448.00 -86.80%
BenchmarkChanProdConsWork0 2014.00 1996.00 -0.89%
BenchmarkChanProdConsWork0-2 1207.00 1127.00 -6.63%
BenchmarkChanProdConsWork0-4 1913.00 611.00 -68.06%
BenchmarkChanProdConsWork0-8 3016.00 949.00 -68.53%
BenchmarkChanProdConsWork0-16 4320.00 1154.00 -73.29%
BenchmarkChanProdConsWork10 1906.00 1897.00 -0.47%
BenchmarkChanProdConsWork10-2 1123.00 1033.00 -8.01%
BenchmarkChanProdConsWork10-4 1076.00 571.00 -46.93%
BenchmarkChanProdConsWork10-8 2748.00 1096.00 -60.12%
BenchmarkChanProdConsWork10-16 4600.00 1105.00 -75.98%
BenchmarkChanProdConsWork100 1884.00 1852.00 -1.70%
BenchmarkChanProdConsWork100-2 1235.00 1146.00 -7.21%
BenchmarkChanProdConsWork100-4 1217.00 619.00 -49.14%
BenchmarkChanProdConsWork100-8 1534.00 509.00 -66.82%
BenchmarkChanProdConsWork100-16 4126.00 918.00 -77.75%
BenchmarkSyscall 34.40 33.30 -3.20%
BenchmarkSyscall-2 160.00 121.00 -24.38%
BenchmarkSyscall-4 131.00 136.00 +3.82%
BenchmarkSyscall-8 139.00 131.00 -5.76%
BenchmarkSyscall-16 161.00 168.00 +4.35%
BenchmarkSyscallWork 950.00 950.00 +0.00%
BenchmarkSyscallWork-2 481.00 480.00 -0.21%
BenchmarkSyscallWork-4 268.00 270.00 +0.75%
BenchmarkSyscallWork-8 156.00 169.00 +8.33%
BenchmarkSyscallWork-16 188.00 184.00 -2.13%
BenchmarkSemaSyntNonblock 36.40 35.60 -2.20%
BenchmarkSemaSyntNonblock-2 81.40 45.10 -44.59%
BenchmarkSemaSyntNonblock-4 126.00 108.00 -14.29%
BenchmarkSemaSyntNonblock-8 112.00 112.00 +0.00%
BenchmarkSemaSyntNonblock-16 110.00 112.00 +1.82%
BenchmarkSemaSyntBlock 35.30 35.30 +0.00%
BenchmarkSemaSyntBlock-2 118.00 124.00 +5.08%
BenchmarkSemaSyntBlock-4 105.00 108.00 +2.86%
BenchmarkSemaSyntBlock-8 101.00 111.00 +9.90%
BenchmarkSemaSyntBlock-16 112.00 118.00 +5.36%
BenchmarkSemaWorkNonblock 810.00 811.00 +0.12%
BenchmarkSemaWorkNonblock-2 476.00 414.00 -13.03%
BenchmarkSemaWorkNonblock-4 238.00 228.00 -4.20%
BenchmarkSemaWorkNonblock-8 140.00 126.00 -10.00%
BenchmarkSemaWorkNonblock-16 117.00 116.00 -0.85%
BenchmarkSemaWorkBlock 810.00 811.00 +0.12%
BenchmarkSemaWorkBlock-2 454.00 466.00 +2.64%
BenchmarkSemaWorkBlock-4 243.00 241.00 -0.82%
BenchmarkSemaWorkBlock-8 145.00 137.00 -5.52%
BenchmarkSemaWorkBlock-16 132.00 123.00 -6.82%
BenchmarkContendedSemaphore 123.00 102.00 -17.07%
BenchmarkContendedSemaphore-2 34.80 34.90 +0.29%
BenchmarkContendedSemaphore-4 34.70 34.80 +0.29%
BenchmarkContendedSemaphore-8 34.70 34.70 +0.00%
BenchmarkContendedSemaphore-16 34.80 34.70 -0.29%
BenchmarkMutex 26.80 26.00 -2.99%
BenchmarkMutex-2 108.00 45.20 -58.15%
BenchmarkMutex-4 103.00 127.00 +23.30%
BenchmarkMutex-8 109.00 147.00 +34.86%
BenchmarkMutex-16 102.00 152.00 +49.02%
BenchmarkMutexSlack 27.00 26.90 -0.37%
BenchmarkMutexSlack-2 149.00 165.00 +10.74%
BenchmarkMutexSlack-4 121.00 209.00 +72.73%
BenchmarkMutexSlack-8 101.00 158.00 +56.44%
BenchmarkMutexSlack-16 97.00 129.00 +32.99%
BenchmarkMutexWork 792.00 794.00 +0.25%
BenchmarkMutexWork-2 407.00 409.00 +0.49%
BenchmarkMutexWork-4 220.00 209.00 -5.00%
BenchmarkMutexWork-8 267.00 160.00 -40.07%
BenchmarkMutexWork-16 315.00 300.00 -4.76%
BenchmarkMutexWorkSlack 792.00 793.00 +0.13%
BenchmarkMutexWorkSlack-2 406.00 404.00 -0.49%
BenchmarkMutexWorkSlack-4 225.00 212.00 -5.78%
BenchmarkMutexWorkSlack-8 268.00 136.00 -49.25%
BenchmarkMutexWorkSlack-16 300.00 300.00 +0.00%
BenchmarkRWMutexWrite100 27.10 27.00 -0.37%
BenchmarkRWMutexWrite100-2 33.10 40.80 +23.26%
BenchmarkRWMutexWrite100-4 113.00 88.10 -22.04%
BenchmarkRWMutexWrite100-8 119.00 95.30 -19.92%
BenchmarkRWMutexWrite100-16 148.00 109.00 -26.35%
BenchmarkRWMutexWrite10 29.60 29.40 -0.68%
BenchmarkRWMutexWrite10-2 111.00 61.40 -44.68%
BenchmarkRWMutexWrite10-4 270.00 208.00 -22.96%
BenchmarkRWMutexWrite10-8 204.00 185.00 -9.31%
BenchmarkRWMutexWrite10-16 261.00 190.00 -27.20%
BenchmarkRWMutexWorkWrite100 1040.00 1036.00 -0.38%
BenchmarkRWMutexWorkWrite100-2 593.00 580.00 -2.19%
BenchmarkRWMutexWorkWrite100-4 470.00 365.00 -22.34%
BenchmarkRWMutexWorkWrite100-8 468.00 289.00 -38.25%
BenchmarkRWMutexWorkWrite100-16 604.00 374.00 -38.08%
BenchmarkRWMutexWorkWrite10 951.00 951.00 +0.00%
BenchmarkRWMutexWorkWrite10-2 1001.00 928.00 -7.29%
BenchmarkRWMutexWorkWrite10-4 1555.00 1006.00 -35.31%
BenchmarkRWMutexWorkWrite10-8 2085.00 1171.00 -43.84%
BenchmarkRWMutexWorkWrite10-16 2082.00 1614.00 -22.48%
R=rsc, iant, msolo, fw, iant
CC=golang-dev
https://golang.org/cl/4711045
2011-07-29 10:44:06 -06:00
|
|
|
again:
|
|
|
|
|
PAUSE
|
|
|
|
|
SUBL $1, AX
|
|
|
|
|
JNZ again
|
|
|
|
|
RET
|
|
|
|
|
|
2015-03-19 17:42:16 -06:00
|
|
|
|
2015-06-15 10:30:23 -06:00
|
|
|
TEXT ·publicationBarrier(SB),NOSPLIT,$0-0
|
|
|
|
|
// Stores are already ordered on x86, so this is just a
|
|
|
|
|
// compile barrier.
|
|
|
|
|
RET
|
|
|
|
|
|
2009-06-03 00:02:12 -06:00
|
|
|
// void jmpdefer(fn, sp);
|
|
|
|
|
// called from deferreturn.
|
2009-01-27 13:03:53 -07:00
|
|
|
// 1. pop the caller
|
|
|
|
|
// 2. sub 5 bytes from the callers return
|
|
|
|
|
// 3. jmp to the argument
|
2013-08-07 15:03:50 -06:00
|
|
|
TEXT runtime·jmpdefer(SB), NOSPLIT, $0-16
|
cmd/cc, runtime: convert C compilers to use Go calling convention
To date, the C compilers and Go compilers differed only in how
values were returned from functions. This made it difficult to call
Go from C or C from Go if return values were involved. It also made
assembly called from Go and assembly called from C different.
This CL changes the C compiler to use the Go conventions, passing
results on the stack, after the arguments.
[Exception: this does not apply to C ... functions, because you can't
know where on the stack the arguments end.]
By doing this, the CL makes it possible to rewrite C functions into Go
one at a time, without worrying about which languages call that
function or which languages it calls.
This CL also updates all the assembly files in package runtime to use
the new conventions. Argument references of the form 40(SP) have
been rewritten to the form name+10(FP) instead, and there are now
Go func prototypes for every assembly function called from C or Go.
This means that 'go vet runtime' checks effectively every assembly
function, and go vet's output was used to automate the bulk of the
conversion.
Some functions, like seek and nsec on Plan 9, needed to be rewritten.
Many assembly routines called from C were reading arguments
incorrectly, using MOVL instead of MOVQ or vice versa, especially on
the less used systems like openbsd.
These were found by go vet and have been corrected too.
If we're lucky, this may reduce flakiness on those systems.
Tested on:
darwin/386
darwin/amd64
linux/arm
linux/386
linux/amd64
If this breaks another system, the bug is almost certainly in the
sys_$GOOS_$GOARCH.s file, since the rest of the CL is tested
by the combination of the above systems.
LGTM=dvyukov, iant
R=golang-codereviews, 0intro, dave, alex.brainman, dvyukov, iant
CC=golang-codereviews, josharian, r
https://golang.org/cl/135830043
2014-08-27 09:32:17 -06:00
|
|
|
MOVQ fv+0(FP), DX // fn
|
|
|
|
|
MOVQ argp+8(FP), BX // caller sp
|
2009-06-03 00:02:12 -06:00
|
|
|
LEAQ -8(BX), SP // caller sp after CALL
|
2016-05-25 18:56:56 -06:00
|
|
|
MOVQ -8(SP), BP // restore BP as if deferreturn returned (harmless if framepointers not in use)
|
2009-06-03 00:02:12 -06:00
|
|
|
SUBQ $5, (SP) // return to CALL again
|
2013-02-22 08:47:54 -07:00
|
|
|
MOVQ 0(DX), BX
|
2013-02-21 15:01:13 -07:00
|
|
|
JMP BX // but first run the deferred function
|
2009-10-03 11:37:12 -06:00
|
|
|
|
2013-06-12 13:22:26 -06:00
|
|
|
// Save state of caller into g->sched. Smashes R8, R9.
|
2013-08-07 11:23:24 -06:00
|
|
|
TEXT gosave<>(SB),NOSPLIT,$0
|
2013-06-12 13:22:26 -06:00
|
|
|
get_tls(R8)
|
|
|
|
|
MOVQ g(R8), R8
|
|
|
|
|
MOVQ 0(SP), R9
|
|
|
|
|
MOVQ R9, (g_sched+gobuf_pc)(R8)
|
|
|
|
|
LEAQ 8(SP), R9
|
|
|
|
|
MOVQ R9, (g_sched+gobuf_sp)(R8)
|
|
|
|
|
MOVQ $0, (g_sched+gobuf_ret)(R8)
|
|
|
|
|
MOVQ $0, (g_sched+gobuf_ctxt)(R8)
|
2015-01-14 09:09:50 -07:00
|
|
|
MOVQ BP, (g_sched+gobuf_bp)(R8)
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
RET
|
|
|
|
|
|
2015-04-27 01:32:23 -06:00
|
|
|
// func asmcgocall(fn, arg unsafe.Pointer) int32
|
2009-10-12 11:26:38 -06:00
|
|
|
// Call fn(arg) on the scheduler stack,
|
|
|
|
|
// aligned appropriately for the gcc ABI.
|
2015-04-27 01:32:23 -06:00
|
|
|
// See cgocall.go for more details.
|
|
|
|
|
TEXT ·asmcgocall(SB),NOSPLIT,$0-20
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ fn+0(FP), AX
|
|
|
|
|
MOVQ arg+8(FP), BX
|
2014-09-03 22:01:55 -06:00
|
|
|
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ SP, DX
|
2009-10-12 11:26:38 -06:00
|
|
|
|
|
|
|
|
// Figure out if we need to switch to m->g0 stack.
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// We get called to create new OS threads too, and those
|
|
|
|
|
// come in on the m->g0 stack already.
|
|
|
|
|
get_tls(CX)
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ g(CX), R8
|
2015-11-19 13:51:39 -07:00
|
|
|
CMPQ R8, $0
|
|
|
|
|
JEQ nosave
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ g_m(R8), R8
|
|
|
|
|
MOVQ m_g0(R8), SI
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ g(CX), DI
|
|
|
|
|
CMPQ SI, DI
|
2014-01-16 21:58:10 -07:00
|
|
|
JEQ nosave
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ m_gsignal(R8), SI
|
2014-01-16 21:58:10 -07:00
|
|
|
CMPQ SI, DI
|
|
|
|
|
JEQ nosave
|
|
|
|
|
|
2015-11-19 13:51:39 -07:00
|
|
|
// Switch to system stack.
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ m_g0(R8), SI
|
2013-06-12 13:22:26 -06:00
|
|
|
CALL gosave<>(SB)
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ SI, g(CX)
|
|
|
|
|
MOVQ (g_sched+gobuf_sp)(SI), SP
|
2009-10-12 11:26:38 -06:00
|
|
|
|
|
|
|
|
// Now on a scheduling stack (a pthread-created stack).
|
2012-09-02 20:12:51 -06:00
|
|
|
// Make sure we have enough room for 4 stack-backed fast-call
|
|
|
|
|
// registers as per windows amd64 calling convention.
|
|
|
|
|
SUBQ $64, SP
|
2009-10-03 11:37:12 -06:00
|
|
|
ANDQ $~15, SP // alignment for gcc ABI
|
2012-09-02 20:12:51 -06:00
|
|
|
MOVQ DI, 48(SP) // save g
|
2014-09-11 21:36:23 -06:00
|
|
|
MOVQ (g_stack+stack_hi)(DI), DI
|
|
|
|
|
SUBQ DX, DI
|
|
|
|
|
MOVQ DI, 40(SP) // save depth in stack (can't just save SP, as stack might be copied during a callback)
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ BX, DI // DI = first argument in AMD64 ABI
|
2011-07-19 08:47:33 -06:00
|
|
|
MOVQ BX, CX // CX = first argument in Win64
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
CALL AX
|
2009-10-12 11:26:38 -06:00
|
|
|
|
2010-08-04 18:50:22 -06:00
|
|
|
// Restore registers, g, stack pointer.
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
get_tls(CX)
|
2012-09-02 20:12:51 -06:00
|
|
|
MOVQ 48(SP), DI
|
2014-09-11 21:36:23 -06:00
|
|
|
MOVQ (g_stack+stack_hi)(DI), SI
|
|
|
|
|
SUBQ 40(SP), SI
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ DI, g(CX)
|
2014-09-11 21:36:23 -06:00
|
|
|
MOVQ SI, SP
|
2015-04-27 01:32:23 -06:00
|
|
|
|
|
|
|
|
MOVL AX, ret+16(FP)
|
2009-10-03 11:37:12 -06:00
|
|
|
RET
|
|
|
|
|
|
2015-11-19 13:51:39 -07:00
|
|
|
nosave:
|
|
|
|
|
// Running on a system stack, perhaps even without a g.
|
|
|
|
|
// Having no g can happen during thread creation or thread teardown
|
|
|
|
|
// (see needm/dropm on Solaris, for example).
|
|
|
|
|
// This code is like the above sequence but without saving/restoring g
|
|
|
|
|
// and without worrying about the stack moving out from under us
|
|
|
|
|
// (because we're on a system stack, not a goroutine stack).
|
|
|
|
|
// The above code could be used directly if already on a system stack,
|
|
|
|
|
// but then the only path through this code would be a rare case on Solaris.
|
|
|
|
|
// Using this code for all "already on system stack" calls exercises it more,
|
|
|
|
|
// which should help keep it correct.
|
|
|
|
|
SUBQ $64, SP
|
|
|
|
|
ANDQ $~15, SP
|
|
|
|
|
MOVQ $0, 48(SP) // where above code stores g, in case someone looks during debugging
|
|
|
|
|
MOVQ DX, 40(SP) // save original stack pointer
|
|
|
|
|
MOVQ BX, DI // DI = first argument in AMD64 ABI
|
|
|
|
|
MOVQ BX, CX // CX = first argument in Win64
|
|
|
|
|
CALL AX
|
|
|
|
|
MOVQ 40(SP), SI // restore original stack pointer
|
|
|
|
|
MOVQ SI, SP
|
|
|
|
|
MOVL AX, ret+16(FP)
|
|
|
|
|
RET
|
|
|
|
|
|
2016-04-27 15:18:29 -06:00
|
|
|
// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt)
|
2013-02-22 14:08:56 -07:00
|
|
|
// Turn the fn into a Go func (by taking its address) and call
|
|
|
|
|
// cgocallback_gofunc.
|
2016-04-27 15:18:29 -06:00
|
|
|
TEXT runtime·cgocallback(SB),NOSPLIT,$32-32
|
2013-02-22 14:08:56 -07:00
|
|
|
LEAQ fn+0(FP), AX
|
|
|
|
|
MOVQ AX, 0(SP)
|
|
|
|
|
MOVQ frame+8(FP), AX
|
|
|
|
|
MOVQ AX, 8(SP)
|
|
|
|
|
MOVQ framesize+16(FP), AX
|
|
|
|
|
MOVQ AX, 16(SP)
|
2016-04-27 15:18:29 -06:00
|
|
|
MOVQ ctxt+24(FP), AX
|
|
|
|
|
MOVQ AX, 24(SP)
|
2013-02-22 14:08:56 -07:00
|
|
|
MOVQ $runtime·cgocallback_gofunc(SB), AX
|
|
|
|
|
CALL AX
|
|
|
|
|
RET
|
|
|
|
|
|
2016-04-27 15:18:29 -06:00
|
|
|
// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt)
|
2015-04-27 01:32:23 -06:00
|
|
|
// See cgocall.go for more details.
|
2016-04-27 15:18:29 -06:00
|
|
|
TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32
|
2014-09-12 05:46:11 -06:00
|
|
|
NO_LOCAL_POINTERS
|
|
|
|
|
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
// If g is nil, Go did not create the current thread.
|
|
|
|
|
// Call needm to obtain one m for temporary use.
|
2013-02-20 15:48:23 -07:00
|
|
|
// In this case, we're running on the thread stack, so there's
|
|
|
|
|
// lots of space, but the linker doesn't know. Hide the call from
|
|
|
|
|
// the linker analysis by using an indirect call through AX.
|
2010-08-04 18:50:22 -06:00
|
|
|
get_tls(CX)
|
2013-02-20 15:48:23 -07:00
|
|
|
#ifdef GOOS_windows
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVL $0, BX
|
2013-02-20 15:48:23 -07:00
|
|
|
CMPQ CX, $0
|
2013-07-23 20:59:32 -06:00
|
|
|
JEQ 2(PC)
|
2013-02-20 15:48:23 -07:00
|
|
|
#endif
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ g(CX), BX
|
|
|
|
|
CMPQ BX, $0
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
JEQ needm
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ g_m(BX), BX
|
|
|
|
|
MOVQ BX, R8 // holds oldm until end of function
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
JMP havem
|
2013-02-20 15:48:23 -07:00
|
|
|
needm:
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
MOVQ $0, 0(SP)
|
2013-02-20 15:48:23 -07:00
|
|
|
MOVQ $runtime·needm(SB), AX
|
|
|
|
|
CALL AX
|
2013-07-24 07:01:57 -06:00
|
|
|
MOVQ 0(SP), R8
|
2013-02-20 15:48:23 -07:00
|
|
|
get_tls(CX)
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ g(CX), BX
|
|
|
|
|
MOVQ g_m(BX), BX
|
2014-10-28 19:53:09 -06:00
|
|
|
|
|
|
|
|
// Set m->sched.sp = SP, so that if a panic happens
|
|
|
|
|
// during the function we are about to execute, it will
|
|
|
|
|
// have a valid SP to run on the g0 stack.
|
|
|
|
|
// The next few lines (after the havem label)
|
|
|
|
|
// will save this SP onto the stack and then write
|
|
|
|
|
// the same SP back to m->sched.sp. That seems redundant,
|
|
|
|
|
// but if an unrecovered panic happens, unwindm will
|
|
|
|
|
// restore the g->sched.sp from the stack location
|
[dev.cc] runtime: delete scalararg, ptrarg; rename onM to systemstack
Scalararg and ptrarg are not "signal safe".
Go code filling them out can be interrupted by a signal,
and then the signal handler runs, and if it also ends up
in Go code that uses scalararg or ptrarg, now the old
values have been smashed.
For the pieces of code that do need to run in a signal handler,
we introduced onM_signalok, which is really just onM
except that the _signalok is meant to convey that the caller
asserts that scalarg and ptrarg will be restored to their old
values after the call (instead of the usual behavior, zeroing them).
Scalararg and ptrarg are also untyped and therefore error-prone.
Go code can always pass a closure instead of using scalararg
and ptrarg; they were only really necessary for C code.
And there's no more C code.
For all these reasons, delete scalararg and ptrarg, converting
the few remaining references to use closures.
Once those are gone, there is no need for a distinction between
onM and onM_signalok, so replace both with a single function
equivalent to the current onM_signalok (that is, it can be called
on any of the curg, g0, and gsignal stacks).
The name onM and the phrase 'm stack' are misnomers,
because on most system an M has two system stacks:
the main thread stack and the signal handling stack.
Correct the misnomer by naming the replacement function systemstack.
Fix a few references to "M stack" in code.
The main motivation for this change is to eliminate scalararg/ptrarg.
Rick and I have already seen them cause problems because
the calling sequence m.ptrarg[0] = p is a heap pointer assignment,
so it gets a write barrier. The write barrier also uses onM, so it has
all the same problems as if it were being invoked by a signal handler.
We worked around this by saving and restoring the old values
and by calling onM_signalok, but there's no point in keeping this nice
home for bugs around any longer.
This CL also changes funcline to return the file name as a result
instead of filling in a passed-in *string. (The *string signature is
left over from when the code was written in and called from C.)
That's arguably an unrelated change, except that once I had done
the ptrarg/scalararg/onM cleanup I started getting false positives
about the *string argument escaping (not allowed in package runtime).
The compiler is wrong, but the easiest fix is to write the code like
Go code instead of like C code. I am a bit worried that the compiler
is wrong because of some use of uninitialized memory in the escape
analysis. If that's the reason, it will go away when we convert the
compiler to Go. (And if not, we'll debug it the next time.)
LGTM=khr
R=r, khr
CC=austin, golang-codereviews, iant, rlh
https://golang.org/cl/174950043
2014-11-12 12:54:31 -07:00
|
|
|
// and then systemstack will try to use it. If we don't set it here,
|
2014-10-28 19:53:09 -06:00
|
|
|
// that restored SP will be uninitialized (typically 0) and
|
|
|
|
|
// will not be usable.
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ m_g0(BX), SI
|
2014-10-28 19:53:09 -06:00
|
|
|
MOVQ SP, (g_sched+gobuf_sp)(SI)
|
2012-03-08 10:12:40 -07:00
|
|
|
|
2013-02-20 15:48:23 -07:00
|
|
|
havem:
|
|
|
|
|
// Now there's a valid m, and we're running on its m->g0.
|
|
|
|
|
// Save current m->g0->sched.sp on stack and then set it to SP.
|
|
|
|
|
// Save current sp in m->g0->sched.sp in preparation for
|
|
|
|
|
// switch back to m->curg stack.
|
2013-07-23 16:40:02 -06:00
|
|
|
// NOTE: unwindm knows that the saved g->sched.sp is at 0(SP).
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ m_g0(BX), SI
|
2013-07-23 16:40:02 -06:00
|
|
|
MOVQ (g_sched+gobuf_sp)(SI), AX
|
|
|
|
|
MOVQ AX, 0(SP)
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ SP, (g_sched+gobuf_sp)(SI)
|
|
|
|
|
|
2013-07-23 16:40:02 -06:00
|
|
|
// Switch to m->curg stack and call runtime.cgocallbackg.
|
|
|
|
|
// Because we are taking over the execution of m->curg
|
|
|
|
|
// but *not* resuming what had been running, we need to
|
|
|
|
|
// save that information (m->curg->sched) so we can restore it.
|
2013-06-05 05:16:53 -06:00
|
|
|
// We can restore m->curg->sched.sp easily, because calling
|
2011-08-18 10:17:09 -06:00
|
|
|
// runtime.cgocallbackg leaves SP unchanged upon return.
|
2013-06-05 05:16:53 -06:00
|
|
|
// To save m->curg->sched.pc, we push it onto the stack.
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// This has the added benefit that it looks to the traceback
|
2011-08-18 10:17:09 -06:00
|
|
|
// routine like cgocallbackg is going to return to that
|
2013-07-23 16:40:02 -06:00
|
|
|
// PC (because the frame we allocate below has the same
|
|
|
|
|
// size as cgocallback_gofunc's frame declared above)
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
// so that the traceback will seamlessly trace back into
|
|
|
|
|
// the earlier calls.
|
2013-07-23 16:40:02 -06:00
|
|
|
//
|
2016-04-27 15:18:29 -06:00
|
|
|
// In the new goroutine, 8(SP) holds the saved R8.
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ m_curg(BX), SI
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ SI, g(CX)
|
|
|
|
|
MOVQ (g_sched+gobuf_sp)(SI), DI // prepare stack as DI
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ (g_sched+gobuf_pc)(SI), BX
|
|
|
|
|
MOVQ BX, -8(DI)
|
2015-01-14 09:09:50 -07:00
|
|
|
// Compute the size of the frame, including return PC and, if
|
|
|
|
|
// GOEXPERIMENT=framepointer, the saved based pointer
|
2016-04-27 15:18:29 -06:00
|
|
|
MOVQ ctxt+24(FP), BX
|
2015-02-19 14:44:06 -07:00
|
|
|
LEAQ fv+0(FP), AX
|
2015-01-14 09:09:50 -07:00
|
|
|
SUBQ SP, AX
|
|
|
|
|
SUBQ AX, DI
|
|
|
|
|
MOVQ DI, SP
|
|
|
|
|
|
2016-04-27 15:18:29 -06:00
|
|
|
MOVQ R8, 8(SP)
|
|
|
|
|
MOVQ BX, 0(SP)
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
CALL runtime·cgocallbackg(SB)
|
2016-04-27 15:18:29 -06:00
|
|
|
MOVQ 8(SP), R8
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
|
2016-03-01 16:21:55 -07:00
|
|
|
// Compute the size of the frame again. FP and SP have
|
2015-01-14 09:09:50 -07:00
|
|
|
// completely different values here than they did above,
|
|
|
|
|
// but only their difference matters.
|
2015-02-19 14:44:06 -07:00
|
|
|
LEAQ fv+0(FP), AX
|
2015-01-14 09:09:50 -07:00
|
|
|
SUBQ SP, AX
|
|
|
|
|
|
2013-06-05 05:16:53 -06:00
|
|
|
// Restore g->sched (== m->curg->sched) from saved values.
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ g(CX), SI
|
2015-01-14 09:09:50 -07:00
|
|
|
MOVQ SP, DI
|
|
|
|
|
ADDQ AX, DI
|
|
|
|
|
MOVQ -8(DI), BX
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ BX, (g_sched+gobuf_pc)(SI)
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ DI, (g_sched+gobuf_sp)(SI)
|
|
|
|
|
|
|
|
|
|
// Switch back to m->g0's stack and restore m->g0->sched.sp.
|
|
|
|
|
// (Unlike m->curg, the g0 goroutine never uses sched.pc,
|
|
|
|
|
// so we do not have to restore it.)
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ g(CX), BX
|
|
|
|
|
MOVQ g_m(BX), BX
|
|
|
|
|
MOVQ m_g0(BX), SI
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
MOVQ SI, g(CX)
|
|
|
|
|
MOVQ (g_sched+gobuf_sp)(SI), SP
|
2013-07-23 16:40:02 -06:00
|
|
|
MOVQ 0(SP), AX
|
|
|
|
|
MOVQ AX, (g_sched+gobuf_sp)(SI)
|
2013-02-20 15:48:23 -07:00
|
|
|
|
|
|
|
|
// If the m on entry was nil, we called needm above to borrow an m
|
|
|
|
|
// for the duration of the call. Since the call is over, return it with dropm.
|
2013-07-24 07:01:57 -06:00
|
|
|
CMPQ R8, $0
|
2013-02-20 15:48:23 -07:00
|
|
|
JNE 3(PC)
|
|
|
|
|
MOVQ $runtime·dropm(SB), AX
|
|
|
|
|
CALL AX
|
runtime: scheduler, cgo reorganization
* Change use of m->g0 stack (aka scheduler stack).
* Provide runtime.mcall(f) to invoke f() on m->g0 stack.
* Replace scheduler loop entry with runtime.mcall(schedule).
Runtime.mcall eliminates the need for fake scheduler states that
exist just to run a bit of code on the m->g0 stack
(Grecovery, Gstackalloc).
The elimination of the scheduler as a loop that stops and
starts using gosave and gogo fixes a bad interaction with the
way cgo uses the m->g0 stack. Cgo runs external (gcc-compiled)
C functions on that stack, and then when calling back into Go,
it sets m->g0->sched.sp below the added call frames, so that
other uses of m->g0's stack will not interfere with those frames.
Unfortunately, gogo (longjmp) back to the scheduler loop at
this point would end up running scheduler with the lower
sp, which no longer points at a valid stack frame for
a call to scheduler. If scheduler then wrote any function call
arguments or local variables to where it expected the stack
frame to be, it would overwrite other data on the stack.
I realized this possibility while debugging a problem with
calling complex Go code in a Go -> C -> Go cgo callback.
This wasn't the bug I was looking for, it turns out, but I believe
it is a real bug nonetheless. Switching to runtime.mcall, which
only adds new frames to the stack and never jumps into
functions running in existing ones, fixes this bug.
* Move cgo-related code out of proc.c into cgocall.c.
* Add very large comment describing cgo call sequences.
* Simpilify, regularize cgo function implementations and names.
* Add test suite as misc/cgo/test.
Now the Go -> C path calls cgocall, which calls asmcgocall,
and the C -> Go path calls cgocallback, which calls cgocallbackg.
The shuffling, which affects mainly the callback case, moves
most of the callback implementation to cgocallback running
on the m->curg stack (not the m->g0 scheduler stack) and
only while accounted for with $GOMAXPROCS (between calls
to exitsyscall and entersyscall).
The previous callback code did not block in startcgocallback's
approximation to exitsyscall, so if, say, the garbage collector
were running, it would still barge in and start doing things
like call malloc. Similarly endcgocallback's approximation of
entersyscall did not call matchmg to kick off new OS threads
when necessary, which caused the bug in issue 1560.
Fixes #1560.
R=iant
CC=golang-dev
https://golang.org/cl/4253054
2011-03-07 08:37:42 -07:00
|
|
|
|
|
|
|
|
// Done!
|
2010-04-09 14:30:11 -06:00
|
|
|
RET
|
|
|
|
|
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
// void setg(G*); set g. for use by needm.
|
cmd/cc, runtime: convert C compilers to use Go calling convention
To date, the C compilers and Go compilers differed only in how
values were returned from functions. This made it difficult to call
Go from C or C from Go if return values were involved. It also made
assembly called from Go and assembly called from C different.
This CL changes the C compiler to use the Go conventions, passing
results on the stack, after the arguments.
[Exception: this does not apply to C ... functions, because you can't
know where on the stack the arguments end.]
By doing this, the CL makes it possible to rewrite C functions into Go
one at a time, without worrying about which languages call that
function or which languages it calls.
This CL also updates all the assembly files in package runtime to use
the new conventions. Argument references of the form 40(SP) have
been rewritten to the form name+10(FP) instead, and there are now
Go func prototypes for every assembly function called from C or Go.
This means that 'go vet runtime' checks effectively every assembly
function, and go vet's output was used to automate the bulk of the
conversion.
Some functions, like seek and nsec on Plan 9, needed to be rewritten.
Many assembly routines called from C were reading arguments
incorrectly, using MOVL instead of MOVQ or vice versa, especially on
the less used systems like openbsd.
These were found by go vet and have been corrected too.
If we're lucky, this may reduce flakiness on those systems.
Tested on:
darwin/386
darwin/amd64
linux/arm
linux/386
linux/amd64
If this breaks another system, the bug is almost certainly in the
sys_$GOOS_$GOARCH.s file, since the rest of the CL is tested
by the combination of the above systems.
LGTM=dvyukov, iant
R=golang-codereviews, 0intro, dave, alex.brainman, dvyukov, iant
CC=golang-codereviews, josharian, r
https://golang.org/cl/135830043
2014-08-27 09:32:17 -06:00
|
|
|
TEXT runtime·setg(SB), NOSPLIT, $0-8
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
MOVQ gg+0(FP), BX
|
2013-02-20 15:48:23 -07:00
|
|
|
#ifdef GOOS_windows
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
CMPQ BX, $0
|
2013-02-20 15:48:23 -07:00
|
|
|
JNE settls
|
|
|
|
|
MOVQ $0, 0x28(GS)
|
|
|
|
|
RET
|
|
|
|
|
settls:
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
MOVQ g_m(BX), AX
|
2013-02-20 15:48:23 -07:00
|
|
|
LEAQ m_tls(AX), AX
|
|
|
|
|
MOVQ AX, 0x28(GS)
|
|
|
|
|
#endif
|
|
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ BX, g(CX)
|
|
|
|
|
RET
|
|
|
|
|
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
// void setg_gcc(G*); set g called from gcc.
|
|
|
|
|
TEXT setg_gcc<>(SB),NOSPLIT,$0
|
2013-03-25 16:14:02 -06:00
|
|
|
get_tls(AX)
|
all: remove 'extern register M *m' from runtime
The runtime has historically held two dedicated values g (current goroutine)
and m (current thread) in 'extern register' slots (TLS on x86, real registers
backed by TLS on ARM).
This CL removes the extern register m; code now uses g->m.
On ARM, this frees up the register that formerly held m (R9).
This is important for NaCl, because NaCl ARM code cannot use R9 at all.
The Go 1 macrobenchmarks (those with per-op times >= 10 µs) are unaffected:
BenchmarkBinaryTree17 5491374955 5471024381 -0.37%
BenchmarkFannkuch11 4357101311 4275174828 -1.88%
BenchmarkGobDecode 11029957 11364184 +3.03%
BenchmarkGobEncode 6852205 6784822 -0.98%
BenchmarkGzip 650795967 650152275 -0.10%
BenchmarkGunzip 140962363 141041670 +0.06%
BenchmarkHTTPClientServer 71581 73081 +2.10%
BenchmarkJSONEncode 31928079 31913356 -0.05%
BenchmarkJSONDecode 117470065 113689916 -3.22%
BenchmarkMandelbrot200 6008923 5998712 -0.17%
BenchmarkGoParse 6310917 6327487 +0.26%
BenchmarkRegexpMatchMedium_1K 114568 114763 +0.17%
BenchmarkRegexpMatchHard_1K 168977 169244 +0.16%
BenchmarkRevcomp 935294971 914060918 -2.27%
BenchmarkTemplate 145917123 148186096 +1.55%
Minux previous reported larger variations, but these were caused by
run-to-run noise, not repeatable slowdowns.
Actual code changes by Minux.
I only did the docs and the benchmarking.
LGTM=dvyukov, iant, minux
R=minux, josharian, iant, dave, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/109050043
2014-06-26 09:54:39 -06:00
|
|
|
MOVQ DI, g(AX)
|
2013-03-25 16:14:02 -06:00
|
|
|
RET
|
|
|
|
|
|
runtime: assume precisestack, copystack, StackCopyAlways, ScanStackByFrames
Commit to stack copying for stack growth.
We're carrying around a surprising amount of cruft from older schemes.
I am confident that precise stack scans and stack copying are here to stay.
Delete fallback code for when precise stack info is disabled.
Delete fallback code for when copying stacks is disabled.
Delete fallback code for when StackCopyAlways is disabled.
Delete Stktop chain - there is only one stack segment now.
Delete M.moreargp, M.moreargsize, M.moreframesize, M.cret.
Delete G.writenbuf (unrelated, just dead).
Delete runtime.lessstack, runtime.oldstack.
Delete many amd64 morestack variants.
Delete initialization of morestack frame/arg sizes (shortens split prologue!).
Replace G's stackguard/stackbase/stack0/stacksize/
syscallstack/syscallguard/forkstackguard with simple stack
bounds (lo, hi).
Update liblink, runtime/cgo for adjustments to G.
LGTM=khr
R=khr, bradfitz
CC=golang-codereviews, iant, r
https://golang.org/cl/137410043
2014-09-09 11:39:57 -06:00
|
|
|
// check that SP is in range [g->stack.lo, g->stack.hi)
|
2013-08-07 11:23:24 -06:00
|
|
|
TEXT runtime·stackcheck(SB), NOSPLIT, $0-0
|
2010-08-04 18:50:22 -06:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ g(CX), AX
|
runtime: assume precisestack, copystack, StackCopyAlways, ScanStackByFrames
Commit to stack copying for stack growth.
We're carrying around a surprising amount of cruft from older schemes.
I am confident that precise stack scans and stack copying are here to stay.
Delete fallback code for when precise stack info is disabled.
Delete fallback code for when copying stacks is disabled.
Delete fallback code for when StackCopyAlways is disabled.
Delete Stktop chain - there is only one stack segment now.
Delete M.moreargp, M.moreargsize, M.moreframesize, M.cret.
Delete G.writenbuf (unrelated, just dead).
Delete runtime.lessstack, runtime.oldstack.
Delete many amd64 morestack variants.
Delete initialization of morestack frame/arg sizes (shortens split prologue!).
Replace G's stackguard/stackbase/stack0/stacksize/
syscallstack/syscallguard/forkstackguard with simple stack
bounds (lo, hi).
Update liblink, runtime/cgo for adjustments to G.
LGTM=khr
R=khr, bradfitz
CC=golang-codereviews, iant, r
https://golang.org/cl/137410043
2014-09-09 11:39:57 -06:00
|
|
|
CMPQ (g_stack+stack_hi)(AX), SP
|
2010-03-30 11:53:16 -06:00
|
|
|
JHI 2(PC)
|
|
|
|
|
INT $3
|
runtime: assume precisestack, copystack, StackCopyAlways, ScanStackByFrames
Commit to stack copying for stack growth.
We're carrying around a surprising amount of cruft from older schemes.
I am confident that precise stack scans and stack copying are here to stay.
Delete fallback code for when precise stack info is disabled.
Delete fallback code for when copying stacks is disabled.
Delete fallback code for when StackCopyAlways is disabled.
Delete Stktop chain - there is only one stack segment now.
Delete M.moreargp, M.moreargsize, M.moreframesize, M.cret.
Delete G.writenbuf (unrelated, just dead).
Delete runtime.lessstack, runtime.oldstack.
Delete many amd64 morestack variants.
Delete initialization of morestack frame/arg sizes (shortens split prologue!).
Replace G's stackguard/stackbase/stack0/stacksize/
syscallstack/syscallguard/forkstackguard with simple stack
bounds (lo, hi).
Update liblink, runtime/cgo for adjustments to G.
LGTM=khr
R=khr, bradfitz
CC=golang-codereviews, iant, r
https://golang.org/cl/137410043
2014-09-09 11:39:57 -06:00
|
|
|
CMPQ SP, (g_stack+stack_lo)(AX)
|
2010-03-30 11:53:16 -06:00
|
|
|
JHI 2(PC)
|
|
|
|
|
INT $3
|
|
|
|
|
RET
|
|
|
|
|
|
runtime: implement GC stack barriers
This commit implements stack barriers to minimize the amount of
stack re-scanning that must be done during mark termination.
Currently the GC scans stacks of active goroutines twice during every
GC cycle: once at the beginning during root discovery and once at the
end during mark termination. The second scan happens while the world
is stopped and guarantees that we've seen all of the roots (since
there are no write barriers on writes to local stack
variables). However, this means pause time is proportional to stack
size. In particularly recursive programs, this can drive pause time up
past our 10ms goal (e.g., it takes about 150ms to scan a 50MB heap).
Re-scanning the entire stack is rarely necessary, especially for large
stacks, because usually most of the frames on the stack were not
active between the first and second scans and hence any changes to
these frames (via non-escaping pointers passed down the stack) were
tracked by write barriers.
To efficiently track how far a stack has been unwound since the first
scan (and, hence, how much needs to be re-scanned), this commit
introduces stack barriers. During the first scan, at exponentially
spaced points in each stack, the scan overwrites return PCs with the
PC of the stack barrier function. When "returned" to, the stack
barrier function records how far the stack has unwound and jumps to
the original return PC for that point in the stack. Then the second
scan only needs to proceed as far as the lowest barrier that hasn't
been hit.
For deeply recursive programs, this substantially reduces mark
termination time (and hence pause time). For the goscheme example
linked in issue #10898, prior to this change, mark termination times
were typically between 100 and 500ms; with this change, mark
termination times are typically between 10 and 20ms. As a result of
the reduced stack scanning work, this reduces overall execution time
of the goscheme example by 20%.
Fixes #10898.
The effect of this on programs that are not deeply recursive is
minimal:
name old time/op new time/op delta
BinaryTree17 3.16s ± 2% 3.26s ± 1% +3.31% (p=0.000 n=19+19)
Fannkuch11 2.42s ± 1% 2.48s ± 1% +2.24% (p=0.000 n=17+19)
FmtFprintfEmpty 50.0ns ± 3% 49.8ns ± 1% ~ (p=0.534 n=20+19)
FmtFprintfString 173ns ± 0% 175ns ± 0% +1.49% (p=0.000 n=16+19)
FmtFprintfInt 170ns ± 1% 175ns ± 1% +2.97% (p=0.000 n=20+19)
FmtFprintfIntInt 288ns ± 0% 295ns ± 0% +2.73% (p=0.000 n=16+19)
FmtFprintfPrefixedInt 242ns ± 1% 252ns ± 1% +4.13% (p=0.000 n=18+18)
FmtFprintfFloat 324ns ± 0% 323ns ± 0% -0.36% (p=0.000 n=20+19)
FmtManyArgs 1.14µs ± 0% 1.12µs ± 1% -1.01% (p=0.000 n=18+19)
GobDecode 8.88ms ± 1% 8.87ms ± 0% ~ (p=0.480 n=19+18)
GobEncode 6.80ms ± 1% 6.85ms ± 0% +0.82% (p=0.000 n=20+18)
Gzip 363ms ± 1% 363ms ± 1% ~ (p=0.077 n=18+20)
Gunzip 90.6ms ± 0% 90.0ms ± 1% -0.71% (p=0.000 n=17+18)
HTTPClientServer 51.5µs ± 1% 50.8µs ± 1% -1.32% (p=0.000 n=18+18)
JSONEncode 17.0ms ± 0% 17.1ms ± 0% +0.40% (p=0.000 n=18+17)
JSONDecode 61.8ms ± 0% 63.8ms ± 1% +3.11% (p=0.000 n=18+17)
Mandelbrot200 3.84ms ± 0% 3.84ms ± 1% ~ (p=0.583 n=19+19)
GoParse 3.71ms ± 1% 3.72ms ± 1% ~ (p=0.159 n=18+19)
RegexpMatchEasy0_32 100ns ± 0% 100ns ± 1% -0.19% (p=0.033 n=17+19)
RegexpMatchEasy0_1K 342ns ± 1% 331ns ± 0% -3.41% (p=0.000 n=19+19)
RegexpMatchEasy1_32 82.5ns ± 0% 81.7ns ± 0% -0.98% (p=0.000 n=18+18)
RegexpMatchEasy1_1K 505ns ± 0% 494ns ± 1% -2.16% (p=0.000 n=18+18)
RegexpMatchMedium_32 137ns ± 1% 137ns ± 1% -0.24% (p=0.048 n=20+18)
RegexpMatchMedium_1K 41.6µs ± 0% 41.3µs ± 1% -0.57% (p=0.004 n=18+20)
RegexpMatchHard_32 2.11µs ± 0% 2.11µs ± 1% +0.20% (p=0.037 n=17+19)
RegexpMatchHard_1K 63.9µs ± 2% 63.3µs ± 0% -0.99% (p=0.000 n=20+17)
Revcomp 560ms ± 1% 522ms ± 0% -6.87% (p=0.000 n=18+16)
Template 75.0ms ± 0% 75.1ms ± 1% +0.18% (p=0.013 n=18+19)
TimeParse 358ns ± 1% 364ns ± 0% +1.74% (p=0.000 n=20+15)
TimeFormat 360ns ± 0% 372ns ± 0% +3.55% (p=0.000 n=20+18)
Change-Id: If8a9bfae6c128d15a4f405e02bcfa50129df82a2
Reviewed-on: https://go-review.googlesource.com/10314
Reviewed-by: Russ Cox <rsc@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
2015-05-20 14:30:49 -06:00
|
|
|
TEXT runtime·getcallerpc(SB),NOSPLIT,$8-16
|
cmd/cc, runtime: convert C compilers to use Go calling convention
To date, the C compilers and Go compilers differed only in how
values were returned from functions. This made it difficult to call
Go from C or C from Go if return values were involved. It also made
assembly called from Go and assembly called from C different.
This CL changes the C compiler to use the Go conventions, passing
results on the stack, after the arguments.
[Exception: this does not apply to C ... functions, because you can't
know where on the stack the arguments end.]
By doing this, the CL makes it possible to rewrite C functions into Go
one at a time, without worrying about which languages call that
function or which languages it calls.
This CL also updates all the assembly files in package runtime to use
the new conventions. Argument references of the form 40(SP) have
been rewritten to the form name+10(FP) instead, and there are now
Go func prototypes for every assembly function called from C or Go.
This means that 'go vet runtime' checks effectively every assembly
function, and go vet's output was used to automate the bulk of the
conversion.
Some functions, like seek and nsec on Plan 9, needed to be rewritten.
Many assembly routines called from C were reading arguments
incorrectly, using MOVL instead of MOVQ or vice versa, especially on
the less used systems like openbsd.
These were found by go vet and have been corrected too.
If we're lucky, this may reduce flakiness on those systems.
Tested on:
darwin/386
darwin/amd64
linux/arm
linux/386
linux/amd64
If this breaks another system, the bug is almost certainly in the
sys_$GOOS_$GOARCH.s file, since the rest of the CL is tested
by the combination of the above systems.
LGTM=dvyukov, iant
R=golang-codereviews, 0intro, dave, alex.brainman, dvyukov, iant
CC=golang-codereviews, josharian, r
https://golang.org/cl/135830043
2014-08-27 09:32:17 -06:00
|
|
|
MOVQ argp+0(FP),AX // addr of first arg
|
2010-04-05 13:51:09 -06:00
|
|
|
MOVQ -8(AX),AX // get calling pc
|
runtime: implement GC stack barriers
This commit implements stack barriers to minimize the amount of
stack re-scanning that must be done during mark termination.
Currently the GC scans stacks of active goroutines twice during every
GC cycle: once at the beginning during root discovery and once at the
end during mark termination. The second scan happens while the world
is stopped and guarantees that we've seen all of the roots (since
there are no write barriers on writes to local stack
variables). However, this means pause time is proportional to stack
size. In particularly recursive programs, this can drive pause time up
past our 10ms goal (e.g., it takes about 150ms to scan a 50MB heap).
Re-scanning the entire stack is rarely necessary, especially for large
stacks, because usually most of the frames on the stack were not
active between the first and second scans and hence any changes to
these frames (via non-escaping pointers passed down the stack) were
tracked by write barriers.
To efficiently track how far a stack has been unwound since the first
scan (and, hence, how much needs to be re-scanned), this commit
introduces stack barriers. During the first scan, at exponentially
spaced points in each stack, the scan overwrites return PCs with the
PC of the stack barrier function. When "returned" to, the stack
barrier function records how far the stack has unwound and jumps to
the original return PC for that point in the stack. Then the second
scan only needs to proceed as far as the lowest barrier that hasn't
been hit.
For deeply recursive programs, this substantially reduces mark
termination time (and hence pause time). For the goscheme example
linked in issue #10898, prior to this change, mark termination times
were typically between 100 and 500ms; with this change, mark
termination times are typically between 10 and 20ms. As a result of
the reduced stack scanning work, this reduces overall execution time
of the goscheme example by 20%.
Fixes #10898.
The effect of this on programs that are not deeply recursive is
minimal:
name old time/op new time/op delta
BinaryTree17 3.16s ± 2% 3.26s ± 1% +3.31% (p=0.000 n=19+19)
Fannkuch11 2.42s ± 1% 2.48s ± 1% +2.24% (p=0.000 n=17+19)
FmtFprintfEmpty 50.0ns ± 3% 49.8ns ± 1% ~ (p=0.534 n=20+19)
FmtFprintfString 173ns ± 0% 175ns ± 0% +1.49% (p=0.000 n=16+19)
FmtFprintfInt 170ns ± 1% 175ns ± 1% +2.97% (p=0.000 n=20+19)
FmtFprintfIntInt 288ns ± 0% 295ns ± 0% +2.73% (p=0.000 n=16+19)
FmtFprintfPrefixedInt 242ns ± 1% 252ns ± 1% +4.13% (p=0.000 n=18+18)
FmtFprintfFloat 324ns ± 0% 323ns ± 0% -0.36% (p=0.000 n=20+19)
FmtManyArgs 1.14µs ± 0% 1.12µs ± 1% -1.01% (p=0.000 n=18+19)
GobDecode 8.88ms ± 1% 8.87ms ± 0% ~ (p=0.480 n=19+18)
GobEncode 6.80ms ± 1% 6.85ms ± 0% +0.82% (p=0.000 n=20+18)
Gzip 363ms ± 1% 363ms ± 1% ~ (p=0.077 n=18+20)
Gunzip 90.6ms ± 0% 90.0ms ± 1% -0.71% (p=0.000 n=17+18)
HTTPClientServer 51.5µs ± 1% 50.8µs ± 1% -1.32% (p=0.000 n=18+18)
JSONEncode 17.0ms ± 0% 17.1ms ± 0% +0.40% (p=0.000 n=18+17)
JSONDecode 61.8ms ± 0% 63.8ms ± 1% +3.11% (p=0.000 n=18+17)
Mandelbrot200 3.84ms ± 0% 3.84ms ± 1% ~ (p=0.583 n=19+19)
GoParse 3.71ms ± 1% 3.72ms ± 1% ~ (p=0.159 n=18+19)
RegexpMatchEasy0_32 100ns ± 0% 100ns ± 1% -0.19% (p=0.033 n=17+19)
RegexpMatchEasy0_1K 342ns ± 1% 331ns ± 0% -3.41% (p=0.000 n=19+19)
RegexpMatchEasy1_32 82.5ns ± 0% 81.7ns ± 0% -0.98% (p=0.000 n=18+18)
RegexpMatchEasy1_1K 505ns ± 0% 494ns ± 1% -2.16% (p=0.000 n=18+18)
RegexpMatchMedium_32 137ns ± 1% 137ns ± 1% -0.24% (p=0.048 n=20+18)
RegexpMatchMedium_1K 41.6µs ± 0% 41.3µs ± 1% -0.57% (p=0.004 n=18+20)
RegexpMatchHard_32 2.11µs ± 0% 2.11µs ± 1% +0.20% (p=0.037 n=17+19)
RegexpMatchHard_1K 63.9µs ± 2% 63.3µs ± 0% -0.99% (p=0.000 n=20+17)
Revcomp 560ms ± 1% 522ms ± 0% -6.87% (p=0.000 n=18+16)
Template 75.0ms ± 0% 75.1ms ± 1% +0.18% (p=0.013 n=18+19)
TimeParse 358ns ± 1% 364ns ± 0% +1.74% (p=0.000 n=20+15)
TimeFormat 360ns ± 0% 372ns ± 0% +3.55% (p=0.000 n=20+18)
Change-Id: If8a9bfae6c128d15a4f405e02bcfa50129df82a2
Reviewed-on: https://go-review.googlesource.com/10314
Reviewed-by: Russ Cox <rsc@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
2015-05-20 14:30:49 -06:00
|
|
|
CMPQ AX, runtime·stackBarrierPC(SB)
|
|
|
|
|
JNE nobar
|
|
|
|
|
// Get original return PC.
|
|
|
|
|
CALL runtime·nextBarrierPC(SB)
|
|
|
|
|
MOVQ 0(SP), AX
|
|
|
|
|
nobar:
|
cmd/cc, runtime: convert C compilers to use Go calling convention
To date, the C compilers and Go compilers differed only in how
values were returned from functions. This made it difficult to call
Go from C or C from Go if return values were involved. It also made
assembly called from Go and assembly called from C different.
This CL changes the C compiler to use the Go conventions, passing
results on the stack, after the arguments.
[Exception: this does not apply to C ... functions, because you can't
know where on the stack the arguments end.]
By doing this, the CL makes it possible to rewrite C functions into Go
one at a time, without worrying about which languages call that
function or which languages it calls.
This CL also updates all the assembly files in package runtime to use
the new conventions. Argument references of the form 40(SP) have
been rewritten to the form name+10(FP) instead, and there are now
Go func prototypes for every assembly function called from C or Go.
This means that 'go vet runtime' checks effectively every assembly
function, and go vet's output was used to automate the bulk of the
conversion.
Some functions, like seek and nsec on Plan 9, needed to be rewritten.
Many assembly routines called from C were reading arguments
incorrectly, using MOVL instead of MOVQ or vice versa, especially on
the less used systems like openbsd.
These were found by go vet and have been corrected too.
If we're lucky, this may reduce flakiness on those systems.
Tested on:
darwin/386
darwin/amd64
linux/arm
linux/386
linux/amd64
If this breaks another system, the bug is almost certainly in the
sys_$GOOS_$GOARCH.s file, since the rest of the CL is tested
by the combination of the above systems.
LGTM=dvyukov, iant
R=golang-codereviews, 0intro, dave, alex.brainman, dvyukov, iant
CC=golang-codereviews, josharian, r
https://golang.org/cl/135830043
2014-08-27 09:32:17 -06:00
|
|
|
MOVQ AX, ret+8(FP)
|
2010-04-05 13:51:09 -06:00
|
|
|
RET
|
|
|
|
|
|
runtime: implement GC stack barriers
This commit implements stack barriers to minimize the amount of
stack re-scanning that must be done during mark termination.
Currently the GC scans stacks of active goroutines twice during every
GC cycle: once at the beginning during root discovery and once at the
end during mark termination. The second scan happens while the world
is stopped and guarantees that we've seen all of the roots (since
there are no write barriers on writes to local stack
variables). However, this means pause time is proportional to stack
size. In particularly recursive programs, this can drive pause time up
past our 10ms goal (e.g., it takes about 150ms to scan a 50MB heap).
Re-scanning the entire stack is rarely necessary, especially for large
stacks, because usually most of the frames on the stack were not
active between the first and second scans and hence any changes to
these frames (via non-escaping pointers passed down the stack) were
tracked by write barriers.
To efficiently track how far a stack has been unwound since the first
scan (and, hence, how much needs to be re-scanned), this commit
introduces stack barriers. During the first scan, at exponentially
spaced points in each stack, the scan overwrites return PCs with the
PC of the stack barrier function. When "returned" to, the stack
barrier function records how far the stack has unwound and jumps to
the original return PC for that point in the stack. Then the second
scan only needs to proceed as far as the lowest barrier that hasn't
been hit.
For deeply recursive programs, this substantially reduces mark
termination time (and hence pause time). For the goscheme example
linked in issue #10898, prior to this change, mark termination times
were typically between 100 and 500ms; with this change, mark
termination times are typically between 10 and 20ms. As a result of
the reduced stack scanning work, this reduces overall execution time
of the goscheme example by 20%.
Fixes #10898.
The effect of this on programs that are not deeply recursive is
minimal:
name old time/op new time/op delta
BinaryTree17 3.16s ± 2% 3.26s ± 1% +3.31% (p=0.000 n=19+19)
Fannkuch11 2.42s ± 1% 2.48s ± 1% +2.24% (p=0.000 n=17+19)
FmtFprintfEmpty 50.0ns ± 3% 49.8ns ± 1% ~ (p=0.534 n=20+19)
FmtFprintfString 173ns ± 0% 175ns ± 0% +1.49% (p=0.000 n=16+19)
FmtFprintfInt 170ns ± 1% 175ns ± 1% +2.97% (p=0.000 n=20+19)
FmtFprintfIntInt 288ns ± 0% 295ns ± 0% +2.73% (p=0.000 n=16+19)
FmtFprintfPrefixedInt 242ns ± 1% 252ns ± 1% +4.13% (p=0.000 n=18+18)
FmtFprintfFloat 324ns ± 0% 323ns ± 0% -0.36% (p=0.000 n=20+19)
FmtManyArgs 1.14µs ± 0% 1.12µs ± 1% -1.01% (p=0.000 n=18+19)
GobDecode 8.88ms ± 1% 8.87ms ± 0% ~ (p=0.480 n=19+18)
GobEncode 6.80ms ± 1% 6.85ms ± 0% +0.82% (p=0.000 n=20+18)
Gzip 363ms ± 1% 363ms ± 1% ~ (p=0.077 n=18+20)
Gunzip 90.6ms ± 0% 90.0ms ± 1% -0.71% (p=0.000 n=17+18)
HTTPClientServer 51.5µs ± 1% 50.8µs ± 1% -1.32% (p=0.000 n=18+18)
JSONEncode 17.0ms ± 0% 17.1ms ± 0% +0.40% (p=0.000 n=18+17)
JSONDecode 61.8ms ± 0% 63.8ms ± 1% +3.11% (p=0.000 n=18+17)
Mandelbrot200 3.84ms ± 0% 3.84ms ± 1% ~ (p=0.583 n=19+19)
GoParse 3.71ms ± 1% 3.72ms ± 1% ~ (p=0.159 n=18+19)
RegexpMatchEasy0_32 100ns ± 0% 100ns ± 1% -0.19% (p=0.033 n=17+19)
RegexpMatchEasy0_1K 342ns ± 1% 331ns ± 0% -3.41% (p=0.000 n=19+19)
RegexpMatchEasy1_32 82.5ns ± 0% 81.7ns ± 0% -0.98% (p=0.000 n=18+18)
RegexpMatchEasy1_1K 505ns ± 0% 494ns ± 1% -2.16% (p=0.000 n=18+18)
RegexpMatchMedium_32 137ns ± 1% 137ns ± 1% -0.24% (p=0.048 n=20+18)
RegexpMatchMedium_1K 41.6µs ± 0% 41.3µs ± 1% -0.57% (p=0.004 n=18+20)
RegexpMatchHard_32 2.11µs ± 0% 2.11µs ± 1% +0.20% (p=0.037 n=17+19)
RegexpMatchHard_1K 63.9µs ± 2% 63.3µs ± 0% -0.99% (p=0.000 n=20+17)
Revcomp 560ms ± 1% 522ms ± 0% -6.87% (p=0.000 n=18+16)
Template 75.0ms ± 0% 75.1ms ± 1% +0.18% (p=0.013 n=18+19)
TimeParse 358ns ± 1% 364ns ± 0% +1.74% (p=0.000 n=20+15)
TimeFormat 360ns ± 0% 372ns ± 0% +3.55% (p=0.000 n=20+18)
Change-Id: If8a9bfae6c128d15a4f405e02bcfa50129df82a2
Reviewed-on: https://go-review.googlesource.com/10314
Reviewed-by: Russ Cox <rsc@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
2015-05-20 14:30:49 -06:00
|
|
|
TEXT runtime·setcallerpc(SB),NOSPLIT,$8-16
|
cmd/cc, runtime: convert C compilers to use Go calling convention
To date, the C compilers and Go compilers differed only in how
values were returned from functions. This made it difficult to call
Go from C or C from Go if return values were involved. It also made
assembly called from Go and assembly called from C different.
This CL changes the C compiler to use the Go conventions, passing
results on the stack, after the arguments.
[Exception: this does not apply to C ... functions, because you can't
know where on the stack the arguments end.]
By doing this, the CL makes it possible to rewrite C functions into Go
one at a time, without worrying about which languages call that
function or which languages it calls.
This CL also updates all the assembly files in package runtime to use
the new conventions. Argument references of the form 40(SP) have
been rewritten to the form name+10(FP) instead, and there are now
Go func prototypes for every assembly function called from C or Go.
This means that 'go vet runtime' checks effectively every assembly
function, and go vet's output was used to automate the bulk of the
conversion.
Some functions, like seek and nsec on Plan 9, needed to be rewritten.
Many assembly routines called from C were reading arguments
incorrectly, using MOVL instead of MOVQ or vice versa, especially on
the less used systems like openbsd.
These were found by go vet and have been corrected too.
If we're lucky, this may reduce flakiness on those systems.
Tested on:
darwin/386
darwin/amd64
linux/arm
linux/386
linux/amd64
If this breaks another system, the bug is almost certainly in the
sys_$GOOS_$GOARCH.s file, since the rest of the CL is tested
by the combination of the above systems.
LGTM=dvyukov, iant
R=golang-codereviews, 0intro, dave, alex.brainman, dvyukov, iant
CC=golang-codereviews, josharian, r
https://golang.org/cl/135830043
2014-08-27 09:32:17 -06:00
|
|
|
MOVQ argp+0(FP),AX // addr of first arg
|
|
|
|
|
MOVQ pc+8(FP), BX
|
runtime: implement GC stack barriers
This commit implements stack barriers to minimize the amount of
stack re-scanning that must be done during mark termination.
Currently the GC scans stacks of active goroutines twice during every
GC cycle: once at the beginning during root discovery and once at the
end during mark termination. The second scan happens while the world
is stopped and guarantees that we've seen all of the roots (since
there are no write barriers on writes to local stack
variables). However, this means pause time is proportional to stack
size. In particularly recursive programs, this can drive pause time up
past our 10ms goal (e.g., it takes about 150ms to scan a 50MB heap).
Re-scanning the entire stack is rarely necessary, especially for large
stacks, because usually most of the frames on the stack were not
active between the first and second scans and hence any changes to
these frames (via non-escaping pointers passed down the stack) were
tracked by write barriers.
To efficiently track how far a stack has been unwound since the first
scan (and, hence, how much needs to be re-scanned), this commit
introduces stack barriers. During the first scan, at exponentially
spaced points in each stack, the scan overwrites return PCs with the
PC of the stack barrier function. When "returned" to, the stack
barrier function records how far the stack has unwound and jumps to
the original return PC for that point in the stack. Then the second
scan only needs to proceed as far as the lowest barrier that hasn't
been hit.
For deeply recursive programs, this substantially reduces mark
termination time (and hence pause time). For the goscheme example
linked in issue #10898, prior to this change, mark termination times
were typically between 100 and 500ms; with this change, mark
termination times are typically between 10 and 20ms. As a result of
the reduced stack scanning work, this reduces overall execution time
of the goscheme example by 20%.
Fixes #10898.
The effect of this on programs that are not deeply recursive is
minimal:
name old time/op new time/op delta
BinaryTree17 3.16s ± 2% 3.26s ± 1% +3.31% (p=0.000 n=19+19)
Fannkuch11 2.42s ± 1% 2.48s ± 1% +2.24% (p=0.000 n=17+19)
FmtFprintfEmpty 50.0ns ± 3% 49.8ns ± 1% ~ (p=0.534 n=20+19)
FmtFprintfString 173ns ± 0% 175ns ± 0% +1.49% (p=0.000 n=16+19)
FmtFprintfInt 170ns ± 1% 175ns ± 1% +2.97% (p=0.000 n=20+19)
FmtFprintfIntInt 288ns ± 0% 295ns ± 0% +2.73% (p=0.000 n=16+19)
FmtFprintfPrefixedInt 242ns ± 1% 252ns ± 1% +4.13% (p=0.000 n=18+18)
FmtFprintfFloat 324ns ± 0% 323ns ± 0% -0.36% (p=0.000 n=20+19)
FmtManyArgs 1.14µs ± 0% 1.12µs ± 1% -1.01% (p=0.000 n=18+19)
GobDecode 8.88ms ± 1% 8.87ms ± 0% ~ (p=0.480 n=19+18)
GobEncode 6.80ms ± 1% 6.85ms ± 0% +0.82% (p=0.000 n=20+18)
Gzip 363ms ± 1% 363ms ± 1% ~ (p=0.077 n=18+20)
Gunzip 90.6ms ± 0% 90.0ms ± 1% -0.71% (p=0.000 n=17+18)
HTTPClientServer 51.5µs ± 1% 50.8µs ± 1% -1.32% (p=0.000 n=18+18)
JSONEncode 17.0ms ± 0% 17.1ms ± 0% +0.40% (p=0.000 n=18+17)
JSONDecode 61.8ms ± 0% 63.8ms ± 1% +3.11% (p=0.000 n=18+17)
Mandelbrot200 3.84ms ± 0% 3.84ms ± 1% ~ (p=0.583 n=19+19)
GoParse 3.71ms ± 1% 3.72ms ± 1% ~ (p=0.159 n=18+19)
RegexpMatchEasy0_32 100ns ± 0% 100ns ± 1% -0.19% (p=0.033 n=17+19)
RegexpMatchEasy0_1K 342ns ± 1% 331ns ± 0% -3.41% (p=0.000 n=19+19)
RegexpMatchEasy1_32 82.5ns ± 0% 81.7ns ± 0% -0.98% (p=0.000 n=18+18)
RegexpMatchEasy1_1K 505ns ± 0% 494ns ± 1% -2.16% (p=0.000 n=18+18)
RegexpMatchMedium_32 137ns ± 1% 137ns ± 1% -0.24% (p=0.048 n=20+18)
RegexpMatchMedium_1K 41.6µs ± 0% 41.3µs ± 1% -0.57% (p=0.004 n=18+20)
RegexpMatchHard_32 2.11µs ± 0% 2.11µs ± 1% +0.20% (p=0.037 n=17+19)
RegexpMatchHard_1K 63.9µs ± 2% 63.3µs ± 0% -0.99% (p=0.000 n=20+17)
Revcomp 560ms ± 1% 522ms ± 0% -6.87% (p=0.000 n=18+16)
Template 75.0ms ± 0% 75.1ms ± 1% +0.18% (p=0.013 n=18+19)
TimeParse 358ns ± 1% 364ns ± 0% +1.74% (p=0.000 n=20+15)
TimeFormat 360ns ± 0% 372ns ± 0% +3.55% (p=0.000 n=20+18)
Change-Id: If8a9bfae6c128d15a4f405e02bcfa50129df82a2
Reviewed-on: https://go-review.googlesource.com/10314
Reviewed-by: Russ Cox <rsc@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
2015-05-20 14:30:49 -06:00
|
|
|
MOVQ -8(AX), CX
|
|
|
|
|
CMPQ CX, runtime·stackBarrierPC(SB)
|
|
|
|
|
JEQ setbar
|
2010-04-05 13:51:09 -06:00
|
|
|
MOVQ BX, -8(AX) // set calling pc
|
|
|
|
|
RET
|
runtime: implement GC stack barriers
This commit implements stack barriers to minimize the amount of
stack re-scanning that must be done during mark termination.
Currently the GC scans stacks of active goroutines twice during every
GC cycle: once at the beginning during root discovery and once at the
end during mark termination. The second scan happens while the world
is stopped and guarantees that we've seen all of the roots (since
there are no write barriers on writes to local stack
variables). However, this means pause time is proportional to stack
size. In particularly recursive programs, this can drive pause time up
past our 10ms goal (e.g., it takes about 150ms to scan a 50MB heap).
Re-scanning the entire stack is rarely necessary, especially for large
stacks, because usually most of the frames on the stack were not
active between the first and second scans and hence any changes to
these frames (via non-escaping pointers passed down the stack) were
tracked by write barriers.
To efficiently track how far a stack has been unwound since the first
scan (and, hence, how much needs to be re-scanned), this commit
introduces stack barriers. During the first scan, at exponentially
spaced points in each stack, the scan overwrites return PCs with the
PC of the stack barrier function. When "returned" to, the stack
barrier function records how far the stack has unwound and jumps to
the original return PC for that point in the stack. Then the second
scan only needs to proceed as far as the lowest barrier that hasn't
been hit.
For deeply recursive programs, this substantially reduces mark
termination time (and hence pause time). For the goscheme example
linked in issue #10898, prior to this change, mark termination times
were typically between 100 and 500ms; with this change, mark
termination times are typically between 10 and 20ms. As a result of
the reduced stack scanning work, this reduces overall execution time
of the goscheme example by 20%.
Fixes #10898.
The effect of this on programs that are not deeply recursive is
minimal:
name old time/op new time/op delta
BinaryTree17 3.16s ± 2% 3.26s ± 1% +3.31% (p=0.000 n=19+19)
Fannkuch11 2.42s ± 1% 2.48s ± 1% +2.24% (p=0.000 n=17+19)
FmtFprintfEmpty 50.0ns ± 3% 49.8ns ± 1% ~ (p=0.534 n=20+19)
FmtFprintfString 173ns ± 0% 175ns ± 0% +1.49% (p=0.000 n=16+19)
FmtFprintfInt 170ns ± 1% 175ns ± 1% +2.97% (p=0.000 n=20+19)
FmtFprintfIntInt 288ns ± 0% 295ns ± 0% +2.73% (p=0.000 n=16+19)
FmtFprintfPrefixedInt 242ns ± 1% 252ns ± 1% +4.13% (p=0.000 n=18+18)
FmtFprintfFloat 324ns ± 0% 323ns ± 0% -0.36% (p=0.000 n=20+19)
FmtManyArgs 1.14µs ± 0% 1.12µs ± 1% -1.01% (p=0.000 n=18+19)
GobDecode 8.88ms ± 1% 8.87ms ± 0% ~ (p=0.480 n=19+18)
GobEncode 6.80ms ± 1% 6.85ms ± 0% +0.82% (p=0.000 n=20+18)
Gzip 363ms ± 1% 363ms ± 1% ~ (p=0.077 n=18+20)
Gunzip 90.6ms ± 0% 90.0ms ± 1% -0.71% (p=0.000 n=17+18)
HTTPClientServer 51.5µs ± 1% 50.8µs ± 1% -1.32% (p=0.000 n=18+18)
JSONEncode 17.0ms ± 0% 17.1ms ± 0% +0.40% (p=0.000 n=18+17)
JSONDecode 61.8ms ± 0% 63.8ms ± 1% +3.11% (p=0.000 n=18+17)
Mandelbrot200 3.84ms ± 0% 3.84ms ± 1% ~ (p=0.583 n=19+19)
GoParse 3.71ms ± 1% 3.72ms ± 1% ~ (p=0.159 n=18+19)
RegexpMatchEasy0_32 100ns ± 0% 100ns ± 1% -0.19% (p=0.033 n=17+19)
RegexpMatchEasy0_1K 342ns ± 1% 331ns ± 0% -3.41% (p=0.000 n=19+19)
RegexpMatchEasy1_32 82.5ns ± 0% 81.7ns ± 0% -0.98% (p=0.000 n=18+18)
RegexpMatchEasy1_1K 505ns ± 0% 494ns ± 1% -2.16% (p=0.000 n=18+18)
RegexpMatchMedium_32 137ns ± 1% 137ns ± 1% -0.24% (p=0.048 n=20+18)
RegexpMatchMedium_1K 41.6µs ± 0% 41.3µs ± 1% -0.57% (p=0.004 n=18+20)
RegexpMatchHard_32 2.11µs ± 0% 2.11µs ± 1% +0.20% (p=0.037 n=17+19)
RegexpMatchHard_1K 63.9µs ± 2% 63.3µs ± 0% -0.99% (p=0.000 n=20+17)
Revcomp 560ms ± 1% 522ms ± 0% -6.87% (p=0.000 n=18+16)
Template 75.0ms ± 0% 75.1ms ± 1% +0.18% (p=0.013 n=18+19)
TimeParse 358ns ± 1% 364ns ± 0% +1.74% (p=0.000 n=20+15)
TimeFormat 360ns ± 0% 372ns ± 0% +3.55% (p=0.000 n=20+18)
Change-Id: If8a9bfae6c128d15a4f405e02bcfa50129df82a2
Reviewed-on: https://go-review.googlesource.com/10314
Reviewed-by: Russ Cox <rsc@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
2015-05-20 14:30:49 -06:00
|
|
|
setbar:
|
|
|
|
|
// Set the stack barrier return PC.
|
|
|
|
|
MOVQ BX, 0(SP)
|
|
|
|
|
CALL runtime·setNextBarrierPC(SB)
|
|
|
|
|
RET
|
2010-04-05 13:51:09 -06:00
|
|
|
|
cmd/cc, runtime: convert C compilers to use Go calling convention
To date, the C compilers and Go compilers differed only in how
values were returned from functions. This made it difficult to call
Go from C or C from Go if return values were involved. It also made
assembly called from Go and assembly called from C different.
This CL changes the C compiler to use the Go conventions, passing
results on the stack, after the arguments.
[Exception: this does not apply to C ... functions, because you can't
know where on the stack the arguments end.]
By doing this, the CL makes it possible to rewrite C functions into Go
one at a time, without worrying about which languages call that
function or which languages it calls.
This CL also updates all the assembly files in package runtime to use
the new conventions. Argument references of the form 40(SP) have
been rewritten to the form name+10(FP) instead, and there are now
Go func prototypes for every assembly function called from C or Go.
This means that 'go vet runtime' checks effectively every assembly
function, and go vet's output was used to automate the bulk of the
conversion.
Some functions, like seek and nsec on Plan 9, needed to be rewritten.
Many assembly routines called from C were reading arguments
incorrectly, using MOVL instead of MOVQ or vice versa, especially on
the less used systems like openbsd.
These were found by go vet and have been corrected too.
If we're lucky, this may reduce flakiness on those systems.
Tested on:
darwin/386
darwin/amd64
linux/arm
linux/386
linux/amd64
If this breaks another system, the bug is almost certainly in the
sys_$GOOS_$GOARCH.s file, since the rest of the CL is tested
by the combination of the above systems.
LGTM=dvyukov, iant
R=golang-codereviews, 0intro, dave, alex.brainman, dvyukov, iant
CC=golang-codereviews, josharian, r
https://golang.org/cl/135830043
2014-08-27 09:32:17 -06:00
|
|
|
TEXT runtime·getcallersp(SB),NOSPLIT,$0-16
|
|
|
|
|
MOVQ argp+0(FP), AX
|
|
|
|
|
MOVQ AX, ret+8(FP)
|
2010-04-05 13:51:09 -06:00
|
|
|
RET
|
|
|
|
|
|
2015-02-17 04:25:49 -07:00
|
|
|
// func cputicks() int64
|
2013-08-07 11:23:24 -06:00
|
|
|
TEXT runtime·cputicks(SB),NOSPLIT,$0-0
|
2015-02-17 04:25:49 -07:00
|
|
|
CMPB runtime·lfenceBeforeRdtsc(SB), $1
|
|
|
|
|
JNE mfence
|
2016-01-13 06:43:22 -07:00
|
|
|
LFENCE
|
2015-02-17 04:25:49 -07:00
|
|
|
JMP done
|
|
|
|
|
mfence:
|
2016-01-13 06:43:22 -07:00
|
|
|
MFENCE
|
2015-02-17 04:25:49 -07:00
|
|
|
done:
|
2012-02-02 12:09:27 -07:00
|
|
|
RDTSC
|
|
|
|
|
SHLQ $32, DX
|
|
|
|
|
ADDQ DX, AX
|
cmd/cc, runtime: convert C compilers to use Go calling convention
To date, the C compilers and Go compilers differed only in how
values were returned from functions. This made it difficult to call
Go from C or C from Go if return values were involved. It also made
assembly called from Go and assembly called from C different.
This CL changes the C compiler to use the Go conventions, passing
results on the stack, after the arguments.
[Exception: this does not apply to C ... functions, because you can't
know where on the stack the arguments end.]
By doing this, the CL makes it possible to rewrite C functions into Go
one at a time, without worrying about which languages call that
function or which languages it calls.
This CL also updates all the assembly files in package runtime to use
the new conventions. Argument references of the form 40(SP) have
been rewritten to the form name+10(FP) instead, and there are now
Go func prototypes for every assembly function called from C or Go.
This means that 'go vet runtime' checks effectively every assembly
function, and go vet's output was used to automate the bulk of the
conversion.
Some functions, like seek and nsec on Plan 9, needed to be rewritten.
Many assembly routines called from C were reading arguments
incorrectly, using MOVL instead of MOVQ or vice versa, especially on
the less used systems like openbsd.
These were found by go vet and have been corrected too.
If we're lucky, this may reduce flakiness on those systems.
Tested on:
darwin/386
darwin/amd64
linux/arm
linux/386
linux/amd64
If this breaks another system, the bug is almost certainly in the
sys_$GOOS_$GOARCH.s file, since the rest of the CL is tested
by the combination of the above systems.
LGTM=dvyukov, iant
R=golang-codereviews, 0intro, dave, alex.brainman, dvyukov, iant
CC=golang-codereviews, josharian, r
https://golang.org/cl/135830043
2014-08-27 09:32:17 -06:00
|
|
|
MOVQ AX, ret+0(FP)
|
2012-02-02 12:09:27 -07:00
|
|
|
RET
|
|
|
|
|
|
2015-01-06 17:42:48 -07:00
|
|
|
// memhash_varlen(p unsafe.Pointer, h seed) uintptr
|
|
|
|
|
// redirects to memhash(p, h, size) using the size
|
|
|
|
|
// stored in the closure.
|
|
|
|
|
TEXT runtime·memhash_varlen(SB),NOSPLIT,$32-24
|
|
|
|
|
GO_ARGS
|
|
|
|
|
NO_LOCAL_POINTERS
|
|
|
|
|
MOVQ p+0(FP), AX
|
|
|
|
|
MOVQ h+8(FP), BX
|
|
|
|
|
MOVQ 8(DX), CX
|
|
|
|
|
MOVQ AX, 0(SP)
|
|
|
|
|
MOVQ BX, 8(SP)
|
|
|
|
|
MOVQ CX, 16(SP)
|
|
|
|
|
CALL runtime·memhash(SB)
|
|
|
|
|
MOVQ 24(SP), AX
|
|
|
|
|
MOVQ AX, ret+16(FP)
|
|
|
|
|
RET
|
|
|
|
|
|
2013-03-12 11:47:44 -06:00
|
|
|
// hash function using AES hardware instructions
|
2014-07-31 16:07:05 -06:00
|
|
|
TEXT runtime·aeshash(SB),NOSPLIT,$0-32
|
|
|
|
|
MOVQ p+0(FP), AX // ptr to data
|
2015-01-06 17:42:48 -07:00
|
|
|
MOVQ s+16(FP), CX // size
|
|
|
|
|
LEAQ ret+24(FP), DX
|
2013-03-12 11:47:44 -06:00
|
|
|
JMP runtime·aeshashbody(SB)
|
|
|
|
|
|
2015-01-06 17:42:48 -07:00
|
|
|
TEXT runtime·aeshashstr(SB),NOSPLIT,$0-24
|
2014-07-31 16:07:05 -06:00
|
|
|
MOVQ p+0(FP), AX // ptr to string struct
|
2013-03-12 11:47:44 -06:00
|
|
|
MOVQ 8(AX), CX // length of string
|
|
|
|
|
MOVQ (AX), AX // string data
|
2015-01-06 17:42:48 -07:00
|
|
|
LEAQ ret+16(FP), DX
|
2013-03-12 11:47:44 -06:00
|
|
|
JMP runtime·aeshashbody(SB)
|
|
|
|
|
|
|
|
|
|
// AX: data
|
|
|
|
|
// CX: length
|
2015-01-06 17:42:48 -07:00
|
|
|
// DX: address to put return value
|
|
|
|
|
TEXT runtime·aeshashbody(SB),NOSPLIT,$0-0
|
2015-08-31 17:26:12 -06:00
|
|
|
// Fill an SSE register with our seeds.
|
|
|
|
|
MOVQ h+8(FP), X0 // 64 bits of per-table hash seed
|
|
|
|
|
PINSRW $4, CX, X0 // 16 bits of length
|
|
|
|
|
PSHUFHW $0, X0, X0 // repeat length 4 times total
|
|
|
|
|
MOVO X0, X1 // save unscrambled seed
|
|
|
|
|
PXOR runtime·aeskeysched(SB), X0 // xor in per-process seed
|
|
|
|
|
AESENC X0, X0 // scramble seed
|
|
|
|
|
|
2013-05-15 10:40:14 -06:00
|
|
|
CMPQ CX, $16
|
2014-12-10 15:20:17 -07:00
|
|
|
JB aes0to15
|
|
|
|
|
JE aes16
|
|
|
|
|
CMPQ CX, $32
|
|
|
|
|
JBE aes17to32
|
|
|
|
|
CMPQ CX, $64
|
|
|
|
|
JBE aes33to64
|
|
|
|
|
CMPQ CX, $128
|
|
|
|
|
JBE aes65to128
|
|
|
|
|
JMP aes129plus
|
|
|
|
|
|
|
|
|
|
aes0to15:
|
2013-03-12 11:47:44 -06:00
|
|
|
TESTQ CX, CX
|
2014-12-10 15:20:17 -07:00
|
|
|
JE aes0
|
2013-03-12 11:47:44 -06:00
|
|
|
|
2014-12-10 15:20:17 -07:00
|
|
|
ADDQ $16, AX
|
|
|
|
|
TESTW $0xff0, AX
|
|
|
|
|
JE endofpage
|
2013-03-12 11:47:44 -06:00
|
|
|
|
2013-05-15 10:40:14 -06:00
|
|
|
// 16 bytes loaded at this address won't cross
|
|
|
|
|
// a page boundary, so we can load it directly.
|
2015-08-31 17:26:12 -06:00
|
|
|
MOVOU -16(AX), X1
|
2013-03-12 11:47:44 -06:00
|
|
|
ADDQ CX, CX
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ $masks<>(SB), AX
|
2015-08-31 17:26:12 -06:00
|
|
|
PAND (AX)(CX*8), X1
|
|
|
|
|
final1:
|
2016-05-26 09:56:49 -06:00
|
|
|
PXOR X0, X1 // xor data with seed
|
|
|
|
|
AESENC X1, X1 // scramble combo 3 times
|
|
|
|
|
AESENC X1, X1
|
2015-08-31 17:26:12 -06:00
|
|
|
AESENC X1, X1
|
|
|
|
|
MOVQ X1, (DX)
|
2014-12-10 15:20:17 -07:00
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
endofpage:
|
2016-03-01 16:21:55 -07:00
|
|
|
// address ends in 1111xxxx. Might be up against
|
2013-03-12 11:47:44 -06:00
|
|
|
// a page boundary, so load ending at last byte.
|
|
|
|
|
// Then shift bytes down using pshufb.
|
2015-08-31 17:26:12 -06:00
|
|
|
MOVOU -32(AX)(CX*1), X1
|
2013-03-12 11:47:44 -06:00
|
|
|
ADDQ CX, CX
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ $shifts<>(SB), AX
|
2015-08-31 17:26:12 -06:00
|
|
|
PSHUFB (AX)(CX*8), X1
|
|
|
|
|
JMP final1
|
2014-12-10 15:20:17 -07:00
|
|
|
|
|
|
|
|
aes0:
|
2015-09-01 13:53:15 -06:00
|
|
|
// Return scrambled input seed
|
2015-08-31 17:26:12 -06:00
|
|
|
AESENC X0, X0
|
|
|
|
|
MOVQ X0, (DX)
|
2013-03-12 11:47:44 -06:00
|
|
|
RET
|
|
|
|
|
|
2014-12-10 15:20:17 -07:00
|
|
|
aes16:
|
2015-08-31 17:26:12 -06:00
|
|
|
MOVOU (AX), X1
|
|
|
|
|
JMP final1
|
2014-12-10 15:20:17 -07:00
|
|
|
|
|
|
|
|
aes17to32:
|
2015-08-31 17:26:12 -06:00
|
|
|
// make second starting seed
|
|
|
|
|
PXOR runtime·aeskeysched+16(SB), X1
|
|
|
|
|
AESENC X1, X1
|
|
|
|
|
|
2014-12-10 15:20:17 -07:00
|
|
|
// load data to be hashed
|
2015-08-31 17:26:12 -06:00
|
|
|
MOVOU (AX), X2
|
|
|
|
|
MOVOU -16(AX)(CX*1), X3
|
2014-12-10 15:20:17 -07:00
|
|
|
|
2016-05-26 09:56:49 -06:00
|
|
|
// xor with seed
|
|
|
|
|
PXOR X0, X2
|
|
|
|
|
PXOR X1, X3
|
|
|
|
|
|
2014-12-10 15:20:17 -07:00
|
|
|
// scramble 3 times
|
2016-05-26 09:56:49 -06:00
|
|
|
AESENC X2, X2
|
|
|
|
|
AESENC X3, X3
|
2015-08-31 17:26:12 -06:00
|
|
|
AESENC X2, X2
|
|
|
|
|
AESENC X3, X3
|
|
|
|
|
AESENC X2, X2
|
|
|
|
|
AESENC X3, X3
|
2014-12-10 15:20:17 -07:00
|
|
|
|
|
|
|
|
// combine results
|
2015-08-31 17:26:12 -06:00
|
|
|
PXOR X3, X2
|
|
|
|
|
MOVQ X2, (DX)
|
2014-12-10 15:20:17 -07:00
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
aes33to64:
|
2015-08-31 17:26:12 -06:00
|
|
|
// make 3 more starting seeds
|
|
|
|
|
MOVO X1, X2
|
|
|
|
|
MOVO X1, X3
|
|
|
|
|
PXOR runtime·aeskeysched+16(SB), X1
|
|
|
|
|
PXOR runtime·aeskeysched+32(SB), X2
|
|
|
|
|
PXOR runtime·aeskeysched+48(SB), X3
|
|
|
|
|
AESENC X1, X1
|
|
|
|
|
AESENC X2, X2
|
|
|
|
|
AESENC X3, X3
|
2014-12-10 15:20:17 -07:00
|
|
|
|
2015-08-31 17:26:12 -06:00
|
|
|
MOVOU (AX), X4
|
|
|
|
|
MOVOU 16(AX), X5
|
|
|
|
|
MOVOU -32(AX)(CX*1), X6
|
|
|
|
|
MOVOU -16(AX)(CX*1), X7
|
2016-05-26 09:56:49 -06:00
|
|
|
|
|
|
|
|
PXOR X0, X4
|
|
|
|
|
PXOR X1, X5
|
|
|
|
|
PXOR X2, X6
|
|
|
|
|
PXOR X3, X7
|
2015-08-31 17:26:12 -06:00
|
|
|
|
2016-05-26 09:56:49 -06:00
|
|
|
AESENC X4, X4
|
|
|
|
|
AESENC X5, X5
|
|
|
|
|
AESENC X6, X6
|
|
|
|
|
AESENC X7, X7
|
2015-08-31 17:26:12 -06:00
|
|
|
|
|
|
|
|
AESENC X4, X4
|
|
|
|
|
AESENC X5, X5
|
|
|
|
|
AESENC X6, X6
|
|
|
|
|
AESENC X7, X7
|
|
|
|
|
|
|
|
|
|
AESENC X4, X4
|
|
|
|
|
AESENC X5, X5
|
|
|
|
|
AESENC X6, X6
|
|
|
|
|
AESENC X7, X7
|
|
|
|
|
|
|
|
|
|
PXOR X6, X4
|
|
|
|
|
PXOR X7, X5
|
|
|
|
|
PXOR X5, X4
|
|
|
|
|
MOVQ X4, (DX)
|
2014-12-10 15:20:17 -07:00
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
aes65to128:
|
2015-08-31 17:26:12 -06:00
|
|
|
// make 7 more starting seeds
|
|
|
|
|
MOVO X1, X2
|
|
|
|
|
MOVO X1, X3
|
|
|
|
|
MOVO X1, X4
|
|
|
|
|
MOVO X1, X5
|
|
|
|
|
MOVO X1, X6
|
|
|
|
|
MOVO X1, X7
|
|
|
|
|
PXOR runtime·aeskeysched+16(SB), X1
|
|
|
|
|
PXOR runtime·aeskeysched+32(SB), X2
|
|
|
|
|
PXOR runtime·aeskeysched+48(SB), X3
|
|
|
|
|
PXOR runtime·aeskeysched+64(SB), X4
|
|
|
|
|
PXOR runtime·aeskeysched+80(SB), X5
|
|
|
|
|
PXOR runtime·aeskeysched+96(SB), X6
|
|
|
|
|
PXOR runtime·aeskeysched+112(SB), X7
|
|
|
|
|
AESENC X1, X1
|
|
|
|
|
AESENC X2, X2
|
|
|
|
|
AESENC X3, X3
|
|
|
|
|
AESENC X4, X4
|
|
|
|
|
AESENC X5, X5
|
|
|
|
|
AESENC X6, X6
|
|
|
|
|
AESENC X7, X7
|
|
|
|
|
|
|
|
|
|
// load data
|
|
|
|
|
MOVOU (AX), X8
|
|
|
|
|
MOVOU 16(AX), X9
|
|
|
|
|
MOVOU 32(AX), X10
|
|
|
|
|
MOVOU 48(AX), X11
|
|
|
|
|
MOVOU -64(AX)(CX*1), X12
|
|
|
|
|
MOVOU -48(AX)(CX*1), X13
|
|
|
|
|
MOVOU -32(AX)(CX*1), X14
|
|
|
|
|
MOVOU -16(AX)(CX*1), X15
|
|
|
|
|
|
2016-05-26 09:56:49 -06:00
|
|
|
// xor with seed
|
|
|
|
|
PXOR X0, X8
|
|
|
|
|
PXOR X1, X9
|
|
|
|
|
PXOR X2, X10
|
|
|
|
|
PXOR X3, X11
|
|
|
|
|
PXOR X4, X12
|
|
|
|
|
PXOR X5, X13
|
|
|
|
|
PXOR X6, X14
|
|
|
|
|
PXOR X7, X15
|
2015-08-31 17:26:12 -06:00
|
|
|
|
2016-05-26 09:56:49 -06:00
|
|
|
// scramble 3 times
|
2015-08-31 17:26:12 -06:00
|
|
|
AESENC X8, X8
|
|
|
|
|
AESENC X9, X9
|
|
|
|
|
AESENC X10, X10
|
|
|
|
|
AESENC X11, X11
|
|
|
|
|
AESENC X12, X12
|
|
|
|
|
AESENC X13, X13
|
|
|
|
|
AESENC X14, X14
|
|
|
|
|
AESENC X15, X15
|
2016-05-26 09:56:49 -06:00
|
|
|
|
|
|
|
|
AESENC X8, X8
|
|
|
|
|
AESENC X9, X9
|
|
|
|
|
AESENC X10, X10
|
|
|
|
|
AESENC X11, X11
|
|
|
|
|
AESENC X12, X12
|
|
|
|
|
AESENC X13, X13
|
|
|
|
|
AESENC X14, X14
|
|
|
|
|
AESENC X15, X15
|
|
|
|
|
|
2015-08-31 17:26:12 -06:00
|
|
|
AESENC X8, X8
|
|
|
|
|
AESENC X9, X9
|
|
|
|
|
AESENC X10, X10
|
|
|
|
|
AESENC X11, X11
|
|
|
|
|
AESENC X12, X12
|
|
|
|
|
AESENC X13, X13
|
|
|
|
|
AESENC X14, X14
|
|
|
|
|
AESENC X15, X15
|
|
|
|
|
|
|
|
|
|
// combine results
|
|
|
|
|
PXOR X12, X8
|
|
|
|
|
PXOR X13, X9
|
|
|
|
|
PXOR X14, X10
|
|
|
|
|
PXOR X15, X11
|
|
|
|
|
PXOR X10, X8
|
|
|
|
|
PXOR X11, X9
|
|
|
|
|
PXOR X9, X8
|
|
|
|
|
MOVQ X8, (DX)
|
2014-12-10 15:20:17 -07:00
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
aes129plus:
|
2015-08-31 17:26:12 -06:00
|
|
|
// make 7 more starting seeds
|
|
|
|
|
MOVO X1, X2
|
|
|
|
|
MOVO X1, X3
|
|
|
|
|
MOVO X1, X4
|
|
|
|
|
MOVO X1, X5
|
|
|
|
|
MOVO X1, X6
|
|
|
|
|
MOVO X1, X7
|
|
|
|
|
PXOR runtime·aeskeysched+16(SB), X1
|
|
|
|
|
PXOR runtime·aeskeysched+32(SB), X2
|
|
|
|
|
PXOR runtime·aeskeysched+48(SB), X3
|
|
|
|
|
PXOR runtime·aeskeysched+64(SB), X4
|
|
|
|
|
PXOR runtime·aeskeysched+80(SB), X5
|
|
|
|
|
PXOR runtime·aeskeysched+96(SB), X6
|
|
|
|
|
PXOR runtime·aeskeysched+112(SB), X7
|
|
|
|
|
AESENC X1, X1
|
|
|
|
|
AESENC X2, X2
|
|
|
|
|
AESENC X3, X3
|
|
|
|
|
AESENC X4, X4
|
|
|
|
|
AESENC X5, X5
|
|
|
|
|
AESENC X6, X6
|
|
|
|
|
AESENC X7, X7
|
|
|
|
|
|
2014-12-10 15:20:17 -07:00
|
|
|
// start with last (possibly overlapping) block
|
2015-08-31 17:26:12 -06:00
|
|
|
MOVOU -128(AX)(CX*1), X8
|
|
|
|
|
MOVOU -112(AX)(CX*1), X9
|
|
|
|
|
MOVOU -96(AX)(CX*1), X10
|
|
|
|
|
MOVOU -80(AX)(CX*1), X11
|
|
|
|
|
MOVOU -64(AX)(CX*1), X12
|
|
|
|
|
MOVOU -48(AX)(CX*1), X13
|
|
|
|
|
MOVOU -32(AX)(CX*1), X14
|
|
|
|
|
MOVOU -16(AX)(CX*1), X15
|
|
|
|
|
|
2016-05-26 09:56:49 -06:00
|
|
|
// xor in seed
|
|
|
|
|
PXOR X0, X8
|
|
|
|
|
PXOR X1, X9
|
|
|
|
|
PXOR X2, X10
|
|
|
|
|
PXOR X3, X11
|
|
|
|
|
PXOR X4, X12
|
|
|
|
|
PXOR X5, X13
|
|
|
|
|
PXOR X6, X14
|
|
|
|
|
PXOR X7, X15
|
2015-08-31 17:26:12 -06:00
|
|
|
|
2014-12-10 15:20:17 -07:00
|
|
|
// compute number of remaining 128-byte blocks
|
|
|
|
|
DECQ CX
|
|
|
|
|
SHRQ $7, CX
|
|
|
|
|
|
|
|
|
|
aesloop:
|
2016-05-26 09:56:49 -06:00
|
|
|
// scramble state
|
|
|
|
|
AESENC X8, X8
|
|
|
|
|
AESENC X9, X9
|
|
|
|
|
AESENC X10, X10
|
|
|
|
|
AESENC X11, X11
|
|
|
|
|
AESENC X12, X12
|
|
|
|
|
AESENC X13, X13
|
|
|
|
|
AESENC X14, X14
|
|
|
|
|
AESENC X15, X15
|
|
|
|
|
|
2014-12-10 15:20:17 -07:00
|
|
|
// scramble state, xor in a block
|
2015-08-31 17:26:12 -06:00
|
|
|
MOVOU (AX), X0
|
|
|
|
|
MOVOU 16(AX), X1
|
|
|
|
|
MOVOU 32(AX), X2
|
|
|
|
|
MOVOU 48(AX), X3
|
|
|
|
|
AESENC X0, X8
|
|
|
|
|
AESENC X1, X9
|
|
|
|
|
AESENC X2, X10
|
|
|
|
|
AESENC X3, X11
|
|
|
|
|
MOVOU 64(AX), X4
|
|
|
|
|
MOVOU 80(AX), X5
|
|
|
|
|
MOVOU 96(AX), X6
|
|
|
|
|
MOVOU 112(AX), X7
|
|
|
|
|
AESENC X4, X12
|
|
|
|
|
AESENC X5, X13
|
|
|
|
|
AESENC X6, X14
|
|
|
|
|
AESENC X7, X15
|
2014-12-10 15:20:17 -07:00
|
|
|
|
2016-05-26 09:56:49 -06:00
|
|
|
ADDQ $128, AX
|
|
|
|
|
DECQ CX
|
|
|
|
|
JNE aesloop
|
|
|
|
|
|
|
|
|
|
// 3 more scrambles to finish
|
2015-08-31 17:26:12 -06:00
|
|
|
AESENC X8, X8
|
|
|
|
|
AESENC X9, X9
|
|
|
|
|
AESENC X10, X10
|
|
|
|
|
AESENC X11, X11
|
|
|
|
|
AESENC X12, X12
|
|
|
|
|
AESENC X13, X13
|
|
|
|
|
AESENC X14, X14
|
|
|
|
|
AESENC X15, X15
|
|
|
|
|
AESENC X8, X8
|
|
|
|
|
AESENC X9, X9
|
|
|
|
|
AESENC X10, X10
|
|
|
|
|
AESENC X11, X11
|
|
|
|
|
AESENC X12, X12
|
|
|
|
|
AESENC X13, X13
|
|
|
|
|
AESENC X14, X14
|
|
|
|
|
AESENC X15, X15
|
|
|
|
|
AESENC X8, X8
|
|
|
|
|
AESENC X9, X9
|
|
|
|
|
AESENC X10, X10
|
|
|
|
|
AESENC X11, X11
|
|
|
|
|
AESENC X12, X12
|
|
|
|
|
AESENC X13, X13
|
|
|
|
|
AESENC X14, X14
|
|
|
|
|
AESENC X15, X15
|
|
|
|
|
|
|
|
|
|
PXOR X12, X8
|
|
|
|
|
PXOR X13, X9
|
|
|
|
|
PXOR X14, X10
|
|
|
|
|
PXOR X15, X11
|
|
|
|
|
PXOR X10, X8
|
|
|
|
|
PXOR X11, X9
|
|
|
|
|
PXOR X9, X8
|
|
|
|
|
MOVQ X8, (DX)
|
2014-12-10 15:20:17 -07:00
|
|
|
RET
|
|
|
|
|
|
2015-01-06 17:42:48 -07:00
|
|
|
TEXT runtime·aeshash32(SB),NOSPLIT,$0-24
|
2014-07-31 16:07:05 -06:00
|
|
|
MOVQ p+0(FP), AX // ptr to data
|
2015-01-06 17:42:48 -07:00
|
|
|
MOVQ h+8(FP), X0 // seed
|
2013-03-12 11:47:44 -06:00
|
|
|
PINSRD $2, (AX), X0 // data
|
2013-03-20 15:34:26 -06:00
|
|
|
AESENC runtime·aeskeysched+0(SB), X0
|
|
|
|
|
AESENC runtime·aeskeysched+16(SB), X0
|
2014-12-10 15:20:17 -07:00
|
|
|
AESENC runtime·aeskeysched+32(SB), X0
|
2015-01-06 17:42:48 -07:00
|
|
|
MOVQ X0, ret+16(FP)
|
2013-03-12 11:47:44 -06:00
|
|
|
RET
|
|
|
|
|
|
2015-01-06 17:42:48 -07:00
|
|
|
TEXT runtime·aeshash64(SB),NOSPLIT,$0-24
|
2014-07-31 16:07:05 -06:00
|
|
|
MOVQ p+0(FP), AX // ptr to data
|
2015-01-06 17:42:48 -07:00
|
|
|
MOVQ h+8(FP), X0 // seed
|
2013-03-12 11:47:44 -06:00
|
|
|
PINSRQ $1, (AX), X0 // data
|
2013-03-20 15:34:26 -06:00
|
|
|
AESENC runtime·aeskeysched+0(SB), X0
|
|
|
|
|
AESENC runtime·aeskeysched+16(SB), X0
|
2014-12-10 15:20:17 -07:00
|
|
|
AESENC runtime·aeskeysched+32(SB), X0
|
2015-01-06 17:42:48 -07:00
|
|
|
MOVQ X0, ret+16(FP)
|
2013-03-12 11:47:44 -06:00
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
// simple mask to get rid of data in the high part of the register.
|
2013-07-16 14:24:09 -06:00
|
|
|
DATA masks<>+0x00(SB)/8, $0x0000000000000000
|
|
|
|
|
DATA masks<>+0x08(SB)/8, $0x0000000000000000
|
|
|
|
|
DATA masks<>+0x10(SB)/8, $0x00000000000000ff
|
|
|
|
|
DATA masks<>+0x18(SB)/8, $0x0000000000000000
|
|
|
|
|
DATA masks<>+0x20(SB)/8, $0x000000000000ffff
|
|
|
|
|
DATA masks<>+0x28(SB)/8, $0x0000000000000000
|
|
|
|
|
DATA masks<>+0x30(SB)/8, $0x0000000000ffffff
|
|
|
|
|
DATA masks<>+0x38(SB)/8, $0x0000000000000000
|
|
|
|
|
DATA masks<>+0x40(SB)/8, $0x00000000ffffffff
|
|
|
|
|
DATA masks<>+0x48(SB)/8, $0x0000000000000000
|
|
|
|
|
DATA masks<>+0x50(SB)/8, $0x000000ffffffffff
|
|
|
|
|
DATA masks<>+0x58(SB)/8, $0x0000000000000000
|
|
|
|
|
DATA masks<>+0x60(SB)/8, $0x0000ffffffffffff
|
|
|
|
|
DATA masks<>+0x68(SB)/8, $0x0000000000000000
|
|
|
|
|
DATA masks<>+0x70(SB)/8, $0x00ffffffffffffff
|
|
|
|
|
DATA masks<>+0x78(SB)/8, $0x0000000000000000
|
|
|
|
|
DATA masks<>+0x80(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA masks<>+0x88(SB)/8, $0x0000000000000000
|
|
|
|
|
DATA masks<>+0x90(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA masks<>+0x98(SB)/8, $0x00000000000000ff
|
|
|
|
|
DATA masks<>+0xa0(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA masks<>+0xa8(SB)/8, $0x000000000000ffff
|
|
|
|
|
DATA masks<>+0xb0(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA masks<>+0xb8(SB)/8, $0x0000000000ffffff
|
|
|
|
|
DATA masks<>+0xc0(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA masks<>+0xc8(SB)/8, $0x00000000ffffffff
|
|
|
|
|
DATA masks<>+0xd0(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA masks<>+0xd8(SB)/8, $0x000000ffffffffff
|
|
|
|
|
DATA masks<>+0xe0(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA masks<>+0xe8(SB)/8, $0x0000ffffffffffff
|
|
|
|
|
DATA masks<>+0xf0(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA masks<>+0xf8(SB)/8, $0x00ffffffffffffff
|
2013-08-07 11:23:24 -06:00
|
|
|
GLOBL masks<>(SB),RODATA,$256
|
2013-07-16 14:24:09 -06:00
|
|
|
|
2015-09-03 00:44:26 -06:00
|
|
|
TEXT ·checkASM(SB),NOSPLIT,$0-1
|
|
|
|
|
// check that masks<>(SB) and shifts<>(SB) are aligned to 16-byte
|
|
|
|
|
MOVQ $masks<>(SB), AX
|
|
|
|
|
MOVQ $shifts<>(SB), BX
|
|
|
|
|
ORQ BX, AX
|
|
|
|
|
TESTQ $15, AX
|
|
|
|
|
SETEQ ret+0(FP)
|
|
|
|
|
RET
|
|
|
|
|
|
2016-03-01 16:21:55 -07:00
|
|
|
// these are arguments to pshufb. They move data down from
|
2013-07-16 14:24:09 -06:00
|
|
|
// the high bytes of the register to the low bytes of the register.
|
|
|
|
|
// index is how many bytes to move.
|
|
|
|
|
DATA shifts<>+0x00(SB)/8, $0x0000000000000000
|
|
|
|
|
DATA shifts<>+0x08(SB)/8, $0x0000000000000000
|
|
|
|
|
DATA shifts<>+0x10(SB)/8, $0xffffffffffffff0f
|
|
|
|
|
DATA shifts<>+0x18(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA shifts<>+0x20(SB)/8, $0xffffffffffff0f0e
|
|
|
|
|
DATA shifts<>+0x28(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA shifts<>+0x30(SB)/8, $0xffffffffff0f0e0d
|
|
|
|
|
DATA shifts<>+0x38(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA shifts<>+0x40(SB)/8, $0xffffffff0f0e0d0c
|
|
|
|
|
DATA shifts<>+0x48(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA shifts<>+0x50(SB)/8, $0xffffff0f0e0d0c0b
|
|
|
|
|
DATA shifts<>+0x58(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA shifts<>+0x60(SB)/8, $0xffff0f0e0d0c0b0a
|
|
|
|
|
DATA shifts<>+0x68(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA shifts<>+0x70(SB)/8, $0xff0f0e0d0c0b0a09
|
|
|
|
|
DATA shifts<>+0x78(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA shifts<>+0x80(SB)/8, $0x0f0e0d0c0b0a0908
|
|
|
|
|
DATA shifts<>+0x88(SB)/8, $0xffffffffffffffff
|
|
|
|
|
DATA shifts<>+0x90(SB)/8, $0x0e0d0c0b0a090807
|
|
|
|
|
DATA shifts<>+0x98(SB)/8, $0xffffffffffffff0f
|
|
|
|
|
DATA shifts<>+0xa0(SB)/8, $0x0d0c0b0a09080706
|
|
|
|
|
DATA shifts<>+0xa8(SB)/8, $0xffffffffffff0f0e
|
|
|
|
|
DATA shifts<>+0xb0(SB)/8, $0x0c0b0a0908070605
|
|
|
|
|
DATA shifts<>+0xb8(SB)/8, $0xffffffffff0f0e0d
|
|
|
|
|
DATA shifts<>+0xc0(SB)/8, $0x0b0a090807060504
|
|
|
|
|
DATA shifts<>+0xc8(SB)/8, $0xffffffff0f0e0d0c
|
|
|
|
|
DATA shifts<>+0xd0(SB)/8, $0x0a09080706050403
|
|
|
|
|
DATA shifts<>+0xd8(SB)/8, $0xffffff0f0e0d0c0b
|
|
|
|
|
DATA shifts<>+0xe0(SB)/8, $0x0908070605040302
|
|
|
|
|
DATA shifts<>+0xe8(SB)/8, $0xffff0f0e0d0c0b0a
|
|
|
|
|
DATA shifts<>+0xf0(SB)/8, $0x0807060504030201
|
|
|
|
|
DATA shifts<>+0xf8(SB)/8, $0xff0f0e0d0c0b0a09
|
2013-08-07 11:23:24 -06:00
|
|
|
GLOBL shifts<>(SB),RODATA,$256
|
2013-07-16 14:24:09 -06:00
|
|
|
|
2016-02-22 14:20:38 -07:00
|
|
|
// memequal(p, q unsafe.Pointer, size uintptr) bool
|
|
|
|
|
TEXT runtime·memequal(SB),NOSPLIT,$0-25
|
2014-07-16 15:16:19 -06:00
|
|
|
MOVQ a+0(FP), SI
|
|
|
|
|
MOVQ b+8(FP), DI
|
2016-02-22 14:20:38 -07:00
|
|
|
CMPQ SI, DI
|
|
|
|
|
JEQ eq
|
2014-07-16 15:16:19 -06:00
|
|
|
MOVQ size+16(FP), BX
|
2015-04-21 15:22:41 -06:00
|
|
|
LEAQ ret+24(FP), AX
|
|
|
|
|
JMP runtime·memeqbody(SB)
|
2016-02-22 14:20:38 -07:00
|
|
|
eq:
|
|
|
|
|
MOVB $1, ret+24(FP)
|
|
|
|
|
RET
|
2014-07-16 15:16:19 -06:00
|
|
|
|
2015-01-06 17:42:48 -07:00
|
|
|
// memequal_varlen(a, b unsafe.Pointer) bool
|
|
|
|
|
TEXT runtime·memequal_varlen(SB),NOSPLIT,$0-17
|
|
|
|
|
MOVQ a+0(FP), SI
|
|
|
|
|
MOVQ b+8(FP), DI
|
|
|
|
|
CMPQ SI, DI
|
|
|
|
|
JEQ eq
|
|
|
|
|
MOVQ 8(DX), BX // compiler stores size at offset 8 in the closure
|
2015-04-21 15:22:41 -06:00
|
|
|
LEAQ ret+16(FP), AX
|
|
|
|
|
JMP runtime·memeqbody(SB)
|
2015-01-06 17:42:48 -07:00
|
|
|
eq:
|
|
|
|
|
MOVB $1, ret+16(FP)
|
|
|
|
|
RET
|
|
|
|
|
|
2014-06-16 22:00:37 -06:00
|
|
|
// eqstring tests whether two strings are equal.
|
2015-02-04 18:31:37 -07:00
|
|
|
// The compiler guarantees that strings passed
|
|
|
|
|
// to eqstring have equal length.
|
2014-06-16 22:00:37 -06:00
|
|
|
// See runtime_test.go:eqstring_generic for
|
2014-08-19 09:50:35 -06:00
|
|
|
// equivalent Go code.
|
2014-06-16 22:00:37 -06:00
|
|
|
TEXT runtime·eqstring(SB),NOSPLIT,$0-33
|
2016-07-11 17:05:57 -06:00
|
|
|
MOVQ s1_base+0(FP), SI
|
|
|
|
|
MOVQ s2_base+16(FP), DI
|
2014-06-16 22:00:37 -06:00
|
|
|
CMPQ SI, DI
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
JEQ eq
|
2016-07-11 17:05:57 -06:00
|
|
|
MOVQ s1_len+8(FP), BX
|
|
|
|
|
LEAQ ret+32(FP), AX
|
2015-04-21 15:22:41 -06:00
|
|
|
JMP runtime·memeqbody(SB)
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
eq:
|
2016-07-11 17:05:57 -06:00
|
|
|
MOVB $1, ret+32(FP)
|
2014-06-16 22:00:37 -06:00
|
|
|
RET
|
|
|
|
|
|
2013-04-02 17:26:15 -06:00
|
|
|
// a in SI
|
|
|
|
|
// b in DI
|
|
|
|
|
// count in BX
|
2015-04-21 15:22:41 -06:00
|
|
|
// address of result byte in AX
|
2013-08-07 11:23:24 -06:00
|
|
|
TEXT runtime·memeqbody(SB),NOSPLIT,$0-0
|
2013-04-02 17:26:15 -06:00
|
|
|
CMPQ BX, $8
|
|
|
|
|
JB small
|
2015-10-29 08:17:05 -06:00
|
|
|
CMPQ BX, $64
|
|
|
|
|
JB bigloop
|
|
|
|
|
CMPB runtime·support_avx2(SB), $1
|
|
|
|
|
JE hugeloop_avx2
|
2013-04-02 17:26:15 -06:00
|
|
|
|
|
|
|
|
// 64 bytes at a time using xmm registers
|
|
|
|
|
hugeloop:
|
|
|
|
|
CMPQ BX, $64
|
|
|
|
|
JB bigloop
|
|
|
|
|
MOVOU (SI), X0
|
|
|
|
|
MOVOU (DI), X1
|
|
|
|
|
MOVOU 16(SI), X2
|
|
|
|
|
MOVOU 16(DI), X3
|
|
|
|
|
MOVOU 32(SI), X4
|
|
|
|
|
MOVOU 32(DI), X5
|
|
|
|
|
MOVOU 48(SI), X6
|
|
|
|
|
MOVOU 48(DI), X7
|
|
|
|
|
PCMPEQB X1, X0
|
|
|
|
|
PCMPEQB X3, X2
|
|
|
|
|
PCMPEQB X5, X4
|
|
|
|
|
PCMPEQB X7, X6
|
|
|
|
|
PAND X2, X0
|
|
|
|
|
PAND X6, X4
|
|
|
|
|
PAND X4, X0
|
|
|
|
|
PMOVMSKB X0, DX
|
|
|
|
|
ADDQ $64, SI
|
|
|
|
|
ADDQ $64, DI
|
|
|
|
|
SUBQ $64, BX
|
|
|
|
|
CMPL DX, $0xffff
|
|
|
|
|
JEQ hugeloop
|
2015-04-21 15:22:41 -06:00
|
|
|
MOVB $0, (AX)
|
2013-04-02 17:26:15 -06:00
|
|
|
RET
|
|
|
|
|
|
2015-10-29 08:17:05 -06:00
|
|
|
// 64 bytes at a time using ymm registers
|
|
|
|
|
hugeloop_avx2:
|
|
|
|
|
CMPQ BX, $64
|
|
|
|
|
JB bigloop_avx2
|
2016-01-22 20:25:15 -07:00
|
|
|
VMOVDQU (SI), Y0
|
|
|
|
|
VMOVDQU (DI), Y1
|
|
|
|
|
VMOVDQU 32(SI), Y2
|
|
|
|
|
VMOVDQU 32(DI), Y3
|
|
|
|
|
VPCMPEQB Y1, Y0, Y4
|
|
|
|
|
VPCMPEQB Y2, Y3, Y5
|
|
|
|
|
VPAND Y4, Y5, Y6
|
|
|
|
|
VPMOVMSKB Y6, DX
|
2015-10-29 08:17:05 -06:00
|
|
|
ADDQ $64, SI
|
|
|
|
|
ADDQ $64, DI
|
|
|
|
|
SUBQ $64, BX
|
|
|
|
|
CMPL DX, $0xffffffff
|
|
|
|
|
JEQ hugeloop_avx2
|
|
|
|
|
VZEROUPPER
|
|
|
|
|
MOVB $0, (AX)
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
bigloop_avx2:
|
|
|
|
|
VZEROUPPER
|
|
|
|
|
|
2013-04-02 17:26:15 -06:00
|
|
|
// 8 bytes at a time using 64-bit register
|
|
|
|
|
bigloop:
|
|
|
|
|
CMPQ BX, $8
|
|
|
|
|
JBE leftover
|
|
|
|
|
MOVQ (SI), CX
|
|
|
|
|
MOVQ (DI), DX
|
|
|
|
|
ADDQ $8, SI
|
|
|
|
|
ADDQ $8, DI
|
|
|
|
|
SUBQ $8, BX
|
|
|
|
|
CMPQ CX, DX
|
|
|
|
|
JEQ bigloop
|
2015-04-21 15:22:41 -06:00
|
|
|
MOVB $0, (AX)
|
2013-04-02 17:26:15 -06:00
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
// remaining 0-8 bytes
|
|
|
|
|
leftover:
|
|
|
|
|
MOVQ -8(SI)(BX*1), CX
|
|
|
|
|
MOVQ -8(DI)(BX*1), DX
|
|
|
|
|
CMPQ CX, DX
|
2015-04-21 15:22:41 -06:00
|
|
|
SETEQ (AX)
|
2013-04-02 17:26:15 -06:00
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
small:
|
|
|
|
|
CMPQ BX, $0
|
|
|
|
|
JEQ equal
|
|
|
|
|
|
|
|
|
|
LEAQ 0(BX*8), CX
|
|
|
|
|
NEGQ CX
|
|
|
|
|
|
|
|
|
|
CMPB SI, $0xf8
|
|
|
|
|
JA si_high
|
|
|
|
|
|
|
|
|
|
// load at SI won't cross a page boundary.
|
|
|
|
|
MOVQ (SI), SI
|
|
|
|
|
JMP si_finish
|
|
|
|
|
si_high:
|
2016-03-01 16:21:55 -07:00
|
|
|
// address ends in 11111xxx. Load up to bytes we want, move to correct position.
|
2013-04-02 17:26:15 -06:00
|
|
|
MOVQ -8(SI)(BX*1), SI
|
|
|
|
|
SHRQ CX, SI
|
|
|
|
|
si_finish:
|
|
|
|
|
|
|
|
|
|
// same for DI.
|
|
|
|
|
CMPB DI, $0xf8
|
|
|
|
|
JA di_high
|
|
|
|
|
MOVQ (DI), DI
|
|
|
|
|
JMP di_finish
|
|
|
|
|
di_high:
|
|
|
|
|
MOVQ -8(DI)(BX*1), DI
|
|
|
|
|
SHRQ CX, DI
|
|
|
|
|
di_finish:
|
|
|
|
|
|
|
|
|
|
SUBQ SI, DI
|
|
|
|
|
SHLQ CX, DI
|
|
|
|
|
equal:
|
2015-04-21 15:22:41 -06:00
|
|
|
SETEQ (AX)
|
2013-04-02 17:26:15 -06:00
|
|
|
RET
|
2013-05-14 17:05:51 -06:00
|
|
|
|
2013-08-07 11:23:24 -06:00
|
|
|
TEXT runtime·cmpstring(SB),NOSPLIT,$0-40
|
cmd/cc, runtime: convert C compilers to use Go calling convention
To date, the C compilers and Go compilers differed only in how
values were returned from functions. This made it difficult to call
Go from C or C from Go if return values were involved. It also made
assembly called from Go and assembly called from C different.
This CL changes the C compiler to use the Go conventions, passing
results on the stack, after the arguments.
[Exception: this does not apply to C ... functions, because you can't
know where on the stack the arguments end.]
By doing this, the CL makes it possible to rewrite C functions into Go
one at a time, without worrying about which languages call that
function or which languages it calls.
This CL also updates all the assembly files in package runtime to use
the new conventions. Argument references of the form 40(SP) have
been rewritten to the form name+10(FP) instead, and there are now
Go func prototypes for every assembly function called from C or Go.
This means that 'go vet runtime' checks effectively every assembly
function, and go vet's output was used to automate the bulk of the
conversion.
Some functions, like seek and nsec on Plan 9, needed to be rewritten.
Many assembly routines called from C were reading arguments
incorrectly, using MOVL instead of MOVQ or vice versa, especially on
the less used systems like openbsd.
These were found by go vet and have been corrected too.
If we're lucky, this may reduce flakiness on those systems.
Tested on:
darwin/386
darwin/amd64
linux/arm
linux/386
linux/amd64
If this breaks another system, the bug is almost certainly in the
sys_$GOOS_$GOARCH.s file, since the rest of the CL is tested
by the combination of the above systems.
LGTM=dvyukov, iant
R=golang-codereviews, 0intro, dave, alex.brainman, dvyukov, iant
CC=golang-codereviews, josharian, r
https://golang.org/cl/135830043
2014-08-27 09:32:17 -06:00
|
|
|
MOVQ s1_base+0(FP), SI
|
|
|
|
|
MOVQ s1_len+8(FP), BX
|
|
|
|
|
MOVQ s2_base+16(FP), DI
|
|
|
|
|
MOVQ s2_len+24(FP), DX
|
2015-04-21 15:22:41 -06:00
|
|
|
LEAQ ret+32(FP), R9
|
|
|
|
|
JMP runtime·cmpbody(SB)
|
2013-05-14 17:05:51 -06:00
|
|
|
|
2014-12-22 11:27:53 -07:00
|
|
|
TEXT bytes·Compare(SB),NOSPLIT,$0-56
|
2013-05-14 17:05:51 -06:00
|
|
|
MOVQ s1+0(FP), SI
|
|
|
|
|
MOVQ s1+8(FP), BX
|
|
|
|
|
MOVQ s2+24(FP), DI
|
|
|
|
|
MOVQ s2+32(FP), DX
|
2015-04-21 15:22:41 -06:00
|
|
|
LEAQ res+48(FP), R9
|
|
|
|
|
JMP runtime·cmpbody(SB)
|
2013-05-14 17:05:51 -06:00
|
|
|
|
|
|
|
|
// input:
|
|
|
|
|
// SI = a
|
|
|
|
|
// DI = b
|
|
|
|
|
// BX = alen
|
|
|
|
|
// DX = blen
|
2015-04-21 15:22:41 -06:00
|
|
|
// R9 = address of output word (stores -1/0/1 here)
|
2013-08-07 11:23:24 -06:00
|
|
|
TEXT runtime·cmpbody(SB),NOSPLIT,$0-0
|
2013-05-14 17:05:51 -06:00
|
|
|
CMPQ SI, DI
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
JEQ allsame
|
2013-05-14 17:05:51 -06:00
|
|
|
CMPQ BX, DX
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ DX, R8
|
|
|
|
|
CMOVQLT BX, R8 // R8 = min(alen, blen) = # of bytes to compare
|
|
|
|
|
CMPQ R8, $8
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
JB small
|
2013-05-14 17:05:51 -06:00
|
|
|
|
2015-07-02 12:43:46 -06:00
|
|
|
CMPQ R8, $63
|
2015-10-28 14:20:26 -06:00
|
|
|
JBE loop
|
|
|
|
|
CMPB runtime·support_avx2(SB), $1
|
|
|
|
|
JEQ big_loop_avx2
|
|
|
|
|
JMP big_loop
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
loop:
|
2015-01-27 16:29:02 -07:00
|
|
|
CMPQ R8, $16
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
JBE _0through16
|
2013-05-14 17:05:51 -06:00
|
|
|
MOVOU (SI), X0
|
|
|
|
|
MOVOU (DI), X1
|
|
|
|
|
PCMPEQB X0, X1
|
|
|
|
|
PMOVMSKB X1, AX
|
|
|
|
|
XORQ $0xffff, AX // convert EQ to NE
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
JNE diff16 // branch if at least one byte is not equal
|
2013-05-14 17:05:51 -06:00
|
|
|
ADDQ $16, SI
|
|
|
|
|
ADDQ $16, DI
|
2015-01-27 16:29:02 -07:00
|
|
|
SUBQ $16, R8
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
JMP loop
|
2013-05-14 17:05:51 -06:00
|
|
|
|
2015-07-02 12:43:46 -06:00
|
|
|
diff64:
|
|
|
|
|
ADDQ $48, SI
|
|
|
|
|
ADDQ $48, DI
|
|
|
|
|
JMP diff16
|
|
|
|
|
diff48:
|
|
|
|
|
ADDQ $32, SI
|
|
|
|
|
ADDQ $32, DI
|
|
|
|
|
JMP diff16
|
|
|
|
|
diff32:
|
|
|
|
|
ADDQ $16, SI
|
|
|
|
|
ADDQ $16, DI
|
2013-05-14 17:05:51 -06:00
|
|
|
// AX = bit mask of differences
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
diff16:
|
2013-05-14 17:05:51 -06:00
|
|
|
BSFQ AX, BX // index of first byte that differs
|
|
|
|
|
XORQ AX, AX
|
|
|
|
|
MOVB (SI)(BX*1), CX
|
|
|
|
|
CMPB CX, (DI)(BX*1)
|
|
|
|
|
SETHI AX
|
|
|
|
|
LEAQ -1(AX*2), AX // convert 1/0 to +1/-1
|
2015-04-21 15:22:41 -06:00
|
|
|
MOVQ AX, (R9)
|
2013-05-14 17:05:51 -06:00
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
// 0 through 16 bytes left, alen>=8, blen>=8
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
_0through16:
|
2015-01-27 16:29:02 -07:00
|
|
|
CMPQ R8, $8
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
JBE _0through8
|
2013-05-14 17:05:51 -06:00
|
|
|
MOVQ (SI), AX
|
|
|
|
|
MOVQ (DI), CX
|
|
|
|
|
CMPQ AX, CX
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
JNE diff8
|
|
|
|
|
_0through8:
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ -8(SI)(R8*1), AX
|
|
|
|
|
MOVQ -8(DI)(R8*1), CX
|
2013-05-14 17:05:51 -06:00
|
|
|
CMPQ AX, CX
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
JEQ allsame
|
2013-05-14 17:05:51 -06:00
|
|
|
|
|
|
|
|
// AX and CX contain parts of a and b that differ.
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
diff8:
|
2013-05-14 17:05:51 -06:00
|
|
|
BSWAPQ AX // reverse order of bytes
|
|
|
|
|
BSWAPQ CX
|
|
|
|
|
XORQ AX, CX
|
|
|
|
|
BSRQ CX, CX // index of highest bit difference
|
|
|
|
|
SHRQ CX, AX // move a's bit to bottom
|
|
|
|
|
ANDQ $1, AX // mask bit
|
|
|
|
|
LEAQ -1(AX*2), AX // 1/0 => +1/-1
|
2015-04-21 15:22:41 -06:00
|
|
|
MOVQ AX, (R9)
|
2013-05-14 17:05:51 -06:00
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
// 0-7 bytes in common
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
small:
|
2015-01-27 16:29:02 -07:00
|
|
|
LEAQ (R8*8), CX // bytes left -> bits left
|
2013-05-14 17:05:51 -06:00
|
|
|
NEGQ CX // - bits lift (== 64 - bits left mod 64)
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
JEQ allsame
|
2013-05-14 17:05:51 -06:00
|
|
|
|
|
|
|
|
// load bytes of a into high bytes of AX
|
|
|
|
|
CMPB SI, $0xf8
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
JA si_high
|
2013-05-14 17:05:51 -06:00
|
|
|
MOVQ (SI), SI
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
JMP si_finish
|
|
|
|
|
si_high:
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ -8(SI)(R8*1), SI
|
2013-05-14 17:05:51 -06:00
|
|
|
SHRQ CX, SI
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
si_finish:
|
2013-05-14 17:05:51 -06:00
|
|
|
SHLQ CX, SI
|
|
|
|
|
|
|
|
|
|
// load bytes of b in to high bytes of BX
|
|
|
|
|
CMPB DI, $0xf8
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
JA di_high
|
2013-05-14 17:05:51 -06:00
|
|
|
MOVQ (DI), DI
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
JMP di_finish
|
|
|
|
|
di_high:
|
2015-01-27 16:29:02 -07:00
|
|
|
MOVQ -8(DI)(R8*1), DI
|
2013-05-14 17:05:51 -06:00
|
|
|
SHRQ CX, DI
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
di_finish:
|
2013-05-14 17:05:51 -06:00
|
|
|
SHLQ CX, DI
|
|
|
|
|
|
|
|
|
|
BSWAPQ SI // reverse order of bytes
|
|
|
|
|
BSWAPQ DI
|
|
|
|
|
XORQ SI, DI // find bit differences
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
JEQ allsame
|
2013-05-14 17:05:51 -06:00
|
|
|
BSRQ DI, CX // index of highest bit difference
|
|
|
|
|
SHRQ CX, SI // move a's bit to bottom
|
|
|
|
|
ANDQ $1, SI // mask bit
|
|
|
|
|
LEAQ -1(SI*2), AX // 1/0 => +1/-1
|
2015-04-21 15:22:41 -06:00
|
|
|
MOVQ AX, (R9)
|
2013-05-14 17:05:51 -06:00
|
|
|
RET
|
|
|
|
|
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
allsame:
|
2013-05-14 17:05:51 -06:00
|
|
|
XORQ AX, AX
|
|
|
|
|
XORQ CX, CX
|
|
|
|
|
CMPQ BX, DX
|
|
|
|
|
SETGT AX // 1 if alen > blen
|
|
|
|
|
SETEQ CX // 1 if alen == blen
|
|
|
|
|
LEAQ -1(CX)(AX*2), AX // 1,0,-1 result
|
2015-04-21 15:22:41 -06:00
|
|
|
MOVQ AX, (R9)
|
2013-05-14 17:05:51 -06:00
|
|
|
RET
|
2013-08-01 17:11:19 -06:00
|
|
|
|
2015-07-02 12:43:46 -06:00
|
|
|
// this works for >= 64 bytes of data.
|
|
|
|
|
big_loop:
|
|
|
|
|
MOVOU (SI), X0
|
|
|
|
|
MOVOU (DI), X1
|
|
|
|
|
PCMPEQB X0, X1
|
|
|
|
|
PMOVMSKB X1, AX
|
|
|
|
|
XORQ $0xffff, AX
|
|
|
|
|
JNE diff16
|
|
|
|
|
|
|
|
|
|
MOVOU 16(SI), X0
|
|
|
|
|
MOVOU 16(DI), X1
|
|
|
|
|
PCMPEQB X0, X1
|
|
|
|
|
PMOVMSKB X1, AX
|
|
|
|
|
XORQ $0xffff, AX
|
|
|
|
|
JNE diff32
|
|
|
|
|
|
|
|
|
|
MOVOU 32(SI), X0
|
|
|
|
|
MOVOU 32(DI), X1
|
|
|
|
|
PCMPEQB X0, X1
|
|
|
|
|
PMOVMSKB X1, AX
|
|
|
|
|
XORQ $0xffff, AX
|
|
|
|
|
JNE diff48
|
|
|
|
|
|
|
|
|
|
MOVOU 48(SI), X0
|
|
|
|
|
MOVOU 48(DI), X1
|
|
|
|
|
PCMPEQB X0, X1
|
|
|
|
|
PMOVMSKB X1, AX
|
|
|
|
|
XORQ $0xffff, AX
|
|
|
|
|
JNE diff64
|
|
|
|
|
|
|
|
|
|
ADDQ $64, SI
|
|
|
|
|
ADDQ $64, DI
|
|
|
|
|
SUBQ $64, R8
|
|
|
|
|
CMPQ R8, $64
|
|
|
|
|
JBE loop
|
|
|
|
|
JMP big_loop
|
|
|
|
|
|
2015-10-28 14:20:26 -06:00
|
|
|
// Compare 64-bytes per loop iteration.
|
|
|
|
|
// Loop is unrolled and uses AVX2.
|
|
|
|
|
big_loop_avx2:
|
2016-01-22 20:25:15 -07:00
|
|
|
VMOVDQU (SI), Y2
|
|
|
|
|
VMOVDQU (DI), Y3
|
|
|
|
|
VMOVDQU 32(SI), Y4
|
|
|
|
|
VMOVDQU 32(DI), Y5
|
|
|
|
|
VPCMPEQB Y2, Y3, Y0
|
|
|
|
|
VPMOVMSKB Y0, AX
|
2015-10-28 14:20:26 -06:00
|
|
|
XORL $0xffffffff, AX
|
|
|
|
|
JNE diff32_avx2
|
2016-01-22 20:25:15 -07:00
|
|
|
VPCMPEQB Y4, Y5, Y6
|
|
|
|
|
VPMOVMSKB Y6, AX
|
2015-10-28 14:20:26 -06:00
|
|
|
XORL $0xffffffff, AX
|
|
|
|
|
JNE diff64_avx2
|
|
|
|
|
|
|
|
|
|
ADDQ $64, SI
|
|
|
|
|
ADDQ $64, DI
|
|
|
|
|
SUBQ $64, R8
|
|
|
|
|
CMPQ R8, $64
|
|
|
|
|
JB big_loop_avx2_exit
|
|
|
|
|
JMP big_loop_avx2
|
|
|
|
|
|
|
|
|
|
// Avoid AVX->SSE transition penalty and search first 32 bytes of 64 byte chunk.
|
|
|
|
|
diff32_avx2:
|
|
|
|
|
VZEROUPPER
|
|
|
|
|
JMP diff16
|
|
|
|
|
|
|
|
|
|
// Same as diff32_avx2, but for last 32 bytes.
|
|
|
|
|
diff64_avx2:
|
|
|
|
|
VZEROUPPER
|
|
|
|
|
JMP diff48
|
|
|
|
|
|
|
|
|
|
// For <64 bytes remainder jump to normal loop.
|
|
|
|
|
big_loop_avx2_exit:
|
|
|
|
|
VZEROUPPER
|
|
|
|
|
JMP loop
|
|
|
|
|
|
|
|
|
|
|
2015-10-28 09:05:05 -06:00
|
|
|
TEXT strings·indexShortStr(SB),NOSPLIT,$0-40
|
|
|
|
|
MOVQ s+0(FP), DI
|
2016-04-21 09:24:12 -06:00
|
|
|
// We want len in DX and AX, because PCMPESTRI implicitly consumes them
|
|
|
|
|
MOVQ s_len+8(FP), DX
|
|
|
|
|
MOVQ c+16(FP), BP
|
|
|
|
|
MOVQ c_len+24(FP), AX
|
2016-04-28 08:34:24 -06:00
|
|
|
MOVQ DI, R10
|
|
|
|
|
LEAQ ret+32(FP), R11
|
|
|
|
|
JMP runtime·indexShortStr(SB)
|
|
|
|
|
|
|
|
|
|
TEXT bytes·indexShortStr(SB),NOSPLIT,$0-56
|
|
|
|
|
MOVQ s+0(FP), DI
|
|
|
|
|
MOVQ s_len+8(FP), DX
|
|
|
|
|
MOVQ c+24(FP), BP
|
|
|
|
|
MOVQ c_len+32(FP), AX
|
|
|
|
|
MOVQ DI, R10
|
|
|
|
|
LEAQ ret+48(FP), R11
|
|
|
|
|
JMP runtime·indexShortStr(SB)
|
|
|
|
|
|
|
|
|
|
// AX: length of string, that we are searching for
|
|
|
|
|
// DX: length of string, in which we are searching
|
|
|
|
|
// DI: pointer to string, in which we are searching
|
|
|
|
|
// BP: pointer to string, that we are searching for
|
|
|
|
|
// R11: address, where to put return value
|
|
|
|
|
TEXT runtime·indexShortStr(SB),NOSPLIT,$0
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ AX, DX
|
2015-10-28 09:05:05 -06:00
|
|
|
JA fail
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ DX, $16
|
|
|
|
|
JAE sse42
|
|
|
|
|
no_sse42:
|
|
|
|
|
CMPQ AX, $2
|
2015-10-28 09:05:05 -06:00
|
|
|
JA _3_or_more
|
2016-04-21 09:24:12 -06:00
|
|
|
MOVW (BP), BP
|
|
|
|
|
LEAQ -1(DI)(DX*1), DX
|
2015-10-28 09:05:05 -06:00
|
|
|
loop2:
|
|
|
|
|
MOVW (DI), SI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPW SI,BP
|
2015-10-28 09:05:05 -06:00
|
|
|
JZ success
|
|
|
|
|
ADDQ $1,DI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ DI,DX
|
2015-10-28 09:05:05 -06:00
|
|
|
JB loop2
|
|
|
|
|
JMP fail
|
|
|
|
|
_3_or_more:
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ AX, $3
|
2015-10-28 09:05:05 -06:00
|
|
|
JA _4_or_more
|
2016-04-21 09:24:12 -06:00
|
|
|
MOVW 1(BP), BX
|
|
|
|
|
MOVW (BP), BP
|
|
|
|
|
LEAQ -2(DI)(DX*1), DX
|
2015-10-28 09:05:05 -06:00
|
|
|
loop3:
|
|
|
|
|
MOVW (DI), SI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPW SI,BP
|
2015-10-28 09:05:05 -06:00
|
|
|
JZ partial_success3
|
|
|
|
|
ADDQ $1,DI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ DI,DX
|
2015-10-28 09:05:05 -06:00
|
|
|
JB loop3
|
|
|
|
|
JMP fail
|
|
|
|
|
partial_success3:
|
|
|
|
|
MOVW 1(DI), SI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPW SI,BX
|
2015-10-28 09:05:05 -06:00
|
|
|
JZ success
|
|
|
|
|
ADDQ $1,DI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ DI,DX
|
2015-10-28 09:05:05 -06:00
|
|
|
JB loop3
|
|
|
|
|
JMP fail
|
|
|
|
|
_4_or_more:
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ AX, $4
|
2015-10-28 09:05:05 -06:00
|
|
|
JA _5_or_more
|
2016-04-21 09:24:12 -06:00
|
|
|
MOVL (BP), BP
|
|
|
|
|
LEAQ -3(DI)(DX*1), DX
|
2015-10-28 09:05:05 -06:00
|
|
|
loop4:
|
|
|
|
|
MOVL (DI), SI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPL SI,BP
|
2015-10-28 09:05:05 -06:00
|
|
|
JZ success
|
|
|
|
|
ADDQ $1,DI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ DI,DX
|
2015-10-28 09:05:05 -06:00
|
|
|
JB loop4
|
|
|
|
|
JMP fail
|
|
|
|
|
_5_or_more:
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ AX, $7
|
2015-10-28 09:05:05 -06:00
|
|
|
JA _8_or_more
|
2016-04-21 09:24:12 -06:00
|
|
|
LEAQ 1(DI)(DX*1), DX
|
|
|
|
|
SUBQ AX, DX
|
|
|
|
|
MOVL -4(BP)(AX*1), BX
|
|
|
|
|
MOVL (BP), BP
|
2015-10-28 09:05:05 -06:00
|
|
|
loop5to7:
|
|
|
|
|
MOVL (DI), SI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPL SI,BP
|
2015-10-28 09:05:05 -06:00
|
|
|
JZ partial_success5to7
|
|
|
|
|
ADDQ $1,DI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ DI,DX
|
2015-10-28 09:05:05 -06:00
|
|
|
JB loop5to7
|
|
|
|
|
JMP fail
|
|
|
|
|
partial_success5to7:
|
2016-04-21 09:24:12 -06:00
|
|
|
MOVL -4(AX)(DI*1), SI
|
|
|
|
|
CMPL SI,BX
|
2015-10-28 09:05:05 -06:00
|
|
|
JZ success
|
|
|
|
|
ADDQ $1,DI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ DI,DX
|
2015-10-28 09:05:05 -06:00
|
|
|
JB loop5to7
|
|
|
|
|
JMP fail
|
|
|
|
|
_8_or_more:
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ AX, $8
|
2015-10-28 09:05:05 -06:00
|
|
|
JA _9_or_more
|
2016-04-21 09:24:12 -06:00
|
|
|
MOVQ (BP), BP
|
|
|
|
|
LEAQ -7(DI)(DX*1), DX
|
2015-10-28 09:05:05 -06:00
|
|
|
loop8:
|
|
|
|
|
MOVQ (DI), SI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ SI,BP
|
2015-10-28 09:05:05 -06:00
|
|
|
JZ success
|
|
|
|
|
ADDQ $1,DI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ DI,DX
|
2015-10-28 09:05:05 -06:00
|
|
|
JB loop8
|
|
|
|
|
JMP fail
|
|
|
|
|
_9_or_more:
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ AX, $16
|
2015-10-28 09:05:05 -06:00
|
|
|
JA _16_or_more
|
2016-04-21 09:24:12 -06:00
|
|
|
LEAQ 1(DI)(DX*1), DX
|
|
|
|
|
SUBQ AX, DX
|
|
|
|
|
MOVQ -8(BP)(AX*1), BX
|
|
|
|
|
MOVQ (BP), BP
|
2015-10-28 09:05:05 -06:00
|
|
|
loop9to15:
|
|
|
|
|
MOVQ (DI), SI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ SI,BP
|
2015-10-28 09:05:05 -06:00
|
|
|
JZ partial_success9to15
|
|
|
|
|
ADDQ $1,DI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ DI,DX
|
2015-10-28 09:05:05 -06:00
|
|
|
JB loop9to15
|
|
|
|
|
JMP fail
|
|
|
|
|
partial_success9to15:
|
2016-04-21 09:24:12 -06:00
|
|
|
MOVQ -8(AX)(DI*1), SI
|
|
|
|
|
CMPQ SI,BX
|
2015-10-28 09:05:05 -06:00
|
|
|
JZ success
|
|
|
|
|
ADDQ $1,DI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ DI,DX
|
2015-10-28 09:05:05 -06:00
|
|
|
JB loop9to15
|
|
|
|
|
JMP fail
|
|
|
|
|
_16_or_more:
|
2016-05-25 07:33:19 -06:00
|
|
|
CMPQ AX, $16
|
2015-10-28 09:05:05 -06:00
|
|
|
JA _17_to_31
|
2016-04-21 09:24:12 -06:00
|
|
|
MOVOU (BP), X1
|
|
|
|
|
LEAQ -15(DI)(DX*1), DX
|
2015-10-28 09:05:05 -06:00
|
|
|
loop16:
|
|
|
|
|
MOVOU (DI), X2
|
|
|
|
|
PCMPEQB X1, X2
|
|
|
|
|
PMOVMSKB X2, SI
|
|
|
|
|
CMPQ SI, $0xffff
|
|
|
|
|
JE success
|
|
|
|
|
ADDQ $1,DI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ DI,DX
|
2015-10-28 09:05:05 -06:00
|
|
|
JB loop16
|
|
|
|
|
JMP fail
|
|
|
|
|
_17_to_31:
|
2016-04-21 09:24:12 -06:00
|
|
|
LEAQ 1(DI)(DX*1), DX
|
|
|
|
|
SUBQ AX, DX
|
|
|
|
|
MOVOU -16(BP)(AX*1), X0
|
|
|
|
|
MOVOU (BP), X1
|
2015-10-28 09:05:05 -06:00
|
|
|
loop17to31:
|
|
|
|
|
MOVOU (DI), X2
|
|
|
|
|
PCMPEQB X1,X2
|
|
|
|
|
PMOVMSKB X2, SI
|
|
|
|
|
CMPQ SI, $0xffff
|
|
|
|
|
JE partial_success17to31
|
|
|
|
|
ADDQ $1,DI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ DI,DX
|
2015-10-28 09:05:05 -06:00
|
|
|
JB loop17to31
|
|
|
|
|
JMP fail
|
|
|
|
|
partial_success17to31:
|
2016-04-21 09:24:12 -06:00
|
|
|
MOVOU -16(AX)(DI*1), X3
|
2015-10-28 09:05:05 -06:00
|
|
|
PCMPEQB X0, X3
|
|
|
|
|
PMOVMSKB X3, SI
|
|
|
|
|
CMPQ SI, $0xffff
|
|
|
|
|
JE success
|
|
|
|
|
ADDQ $1,DI
|
2016-04-21 09:24:12 -06:00
|
|
|
CMPQ DI,DX
|
2015-10-28 09:05:05 -06:00
|
|
|
JB loop17to31
|
|
|
|
|
fail:
|
2016-04-28 08:34:24 -06:00
|
|
|
MOVQ $-1, (R11)
|
2015-10-28 09:05:05 -06:00
|
|
|
RET
|
2016-04-21 09:24:12 -06:00
|
|
|
sse42:
|
|
|
|
|
MOVL runtime·cpuid_ecx(SB), CX
|
|
|
|
|
ANDL $0x100000, CX
|
|
|
|
|
JZ no_sse42
|
|
|
|
|
CMPQ AX, $12
|
|
|
|
|
// PCMPESTRI is slower than normal compare,
|
|
|
|
|
// so using it makes sense only if we advance 4+ bytes per compare
|
|
|
|
|
// This value was determined experimentally and is the ~same
|
|
|
|
|
// on Nehalem (first with SSE42) and Haswell.
|
|
|
|
|
JAE _9_or_more
|
|
|
|
|
LEAQ 16(BP), SI
|
|
|
|
|
TESTW $0xff0, SI
|
|
|
|
|
JEQ no_sse42
|
|
|
|
|
MOVOU (BP), X1
|
|
|
|
|
LEAQ -15(DI)(DX*1), SI
|
|
|
|
|
MOVQ $16, R9
|
|
|
|
|
SUBQ AX, R9 // We advance by 16-len(sep) each iteration, so precalculate it into R9
|
|
|
|
|
loop_sse42:
|
|
|
|
|
// 0x0c means: unsigned byte compare (bits 0,1 are 00)
|
|
|
|
|
// for equality (bits 2,3 are 11)
|
|
|
|
|
// result is not masked or inverted (bits 4,5 are 00)
|
|
|
|
|
// and corresponds to first matching byte (bit 6 is 0)
|
|
|
|
|
PCMPESTRI $0x0c, (DI), X1
|
|
|
|
|
// CX == 16 means no match,
|
|
|
|
|
// CX > R9 means partial match at the end of the string,
|
|
|
|
|
// otherwise sep is at offset CX from X1 start
|
|
|
|
|
CMPQ CX, R9
|
|
|
|
|
JBE sse42_success
|
|
|
|
|
ADDQ R9, DI
|
|
|
|
|
CMPQ DI, SI
|
|
|
|
|
JB loop_sse42
|
|
|
|
|
PCMPESTRI $0x0c, -1(SI), X1
|
|
|
|
|
CMPQ CX, R9
|
|
|
|
|
JA fail
|
|
|
|
|
LEAQ -1(SI), DI
|
|
|
|
|
sse42_success:
|
|
|
|
|
ADDQ CX, DI
|
2015-10-28 09:05:05 -06:00
|
|
|
success:
|
2016-04-28 08:34:24 -06:00
|
|
|
SUBQ R10, DI
|
|
|
|
|
MOVQ DI, (R11)
|
2015-10-28 09:05:05 -06:00
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
|
2015-03-06 22:18:16 -07:00
|
|
|
TEXT bytes·IndexByte(SB),NOSPLIT,$0-40
|
2013-08-01 17:11:19 -06:00
|
|
|
MOVQ s+0(FP), SI
|
|
|
|
|
MOVQ s_len+8(FP), BX
|
|
|
|
|
MOVB c+24(FP), AL
|
2015-04-21 15:22:41 -06:00
|
|
|
LEAQ ret+32(FP), R8
|
|
|
|
|
JMP runtime·indexbytebody(SB)
|
2013-08-05 16:04:05 -06:00
|
|
|
|
2015-03-06 22:18:16 -07:00
|
|
|
TEXT strings·IndexByte(SB),NOSPLIT,$0-32
|
2013-08-05 16:04:05 -06:00
|
|
|
MOVQ s+0(FP), SI
|
|
|
|
|
MOVQ s_len+8(FP), BX
|
|
|
|
|
MOVB c+16(FP), AL
|
2015-04-21 15:22:41 -06:00
|
|
|
LEAQ ret+24(FP), R8
|
|
|
|
|
JMP runtime·indexbytebody(SB)
|
2013-08-05 16:04:05 -06:00
|
|
|
|
|
|
|
|
// input:
|
|
|
|
|
// SI: data
|
|
|
|
|
// BX: data len
|
|
|
|
|
// AL: byte sought
|
2015-04-21 15:22:41 -06:00
|
|
|
// R8: address to put result
|
2013-08-07 11:23:24 -06:00
|
|
|
TEXT runtime·indexbytebody(SB),NOSPLIT,$0
|
2016-01-15 19:17:09 -07:00
|
|
|
// Shuffle X0 around so that each byte contains
|
|
|
|
|
// the character we're looking for.
|
2013-08-01 17:11:19 -06:00
|
|
|
MOVD AX, X0
|
|
|
|
|
PUNPCKLBW X0, X0
|
|
|
|
|
PUNPCKLBW X0, X0
|
|
|
|
|
PSHUFL $0, X0, X0
|
2016-01-15 19:17:09 -07:00
|
|
|
|
|
|
|
|
CMPQ BX, $16
|
|
|
|
|
JLT small
|
|
|
|
|
|
|
|
|
|
MOVQ SI, DI
|
2013-08-01 17:11:19 -06:00
|
|
|
|
2016-01-15 19:17:09 -07:00
|
|
|
CMPQ BX, $32
|
|
|
|
|
JA avx2
|
2013-08-01 17:11:19 -06:00
|
|
|
sse:
|
2016-01-15 19:17:09 -07:00
|
|
|
LEAQ -16(SI)(BX*1), AX // AX = address of last 16 bytes
|
|
|
|
|
JMP sseloopentry
|
|
|
|
|
|
|
|
|
|
sseloop:
|
|
|
|
|
// Move the next 16-byte chunk of the data into X1.
|
|
|
|
|
MOVOU (DI), X1
|
|
|
|
|
// Compare bytes in X0 to X1.
|
|
|
|
|
PCMPEQB X0, X1
|
|
|
|
|
// Take the top bit of each byte in X1 and put the result in DX.
|
2013-08-01 17:11:19 -06:00
|
|
|
PMOVMSKB X1, DX
|
2016-01-15 19:17:09 -07:00
|
|
|
// Find first set bit, if any.
|
|
|
|
|
BSFL DX, DX
|
|
|
|
|
JNZ ssesuccess
|
|
|
|
|
// Advance to next block.
|
|
|
|
|
ADDQ $16, DI
|
|
|
|
|
sseloopentry:
|
|
|
|
|
CMPQ DI, AX
|
|
|
|
|
JB sseloop
|
2013-08-01 17:11:19 -06:00
|
|
|
|
2016-03-01 16:21:55 -07:00
|
|
|
// Search the last 16-byte chunk. This chunk may overlap with the
|
2016-01-15 19:17:09 -07:00
|
|
|
// chunks we've already searched, but that's ok.
|
|
|
|
|
MOVQ AX, DI
|
|
|
|
|
MOVOU (AX), X1
|
|
|
|
|
PCMPEQB X0, X1
|
|
|
|
|
PMOVMSKB X1, DX
|
|
|
|
|
BSFL DX, DX
|
|
|
|
|
JNZ ssesuccess
|
2013-08-01 17:11:19 -06:00
|
|
|
|
|
|
|
|
failure:
|
2015-04-21 15:22:41 -06:00
|
|
|
MOVQ $-1, (R8)
|
2013-08-01 17:11:19 -06:00
|
|
|
RET
|
|
|
|
|
|
2016-01-15 19:17:09 -07:00
|
|
|
// We've found a chunk containing the byte.
|
|
|
|
|
// The chunk was loaded from DI.
|
|
|
|
|
// The index of the matching byte in the chunk is DX.
|
|
|
|
|
// The start of the data is SI.
|
|
|
|
|
ssesuccess:
|
|
|
|
|
SUBQ SI, DI // Compute offset of chunk within data.
|
|
|
|
|
ADDQ DX, DI // Add offset of byte within chunk.
|
|
|
|
|
MOVQ DI, (R8)
|
|
|
|
|
RET
|
|
|
|
|
|
2013-08-01 17:11:19 -06:00
|
|
|
// handle for lengths < 16
|
[dev.power64] cmd/5a, cmd/6a, cmd/8a, cmd/9a: make labels function-scoped
I removed support for jumping between functions years ago,
as part of doing the instruction layout for each function separately.
Given that, it makes sense to treat labels as function-scoped.
This lets each function have its own 'loop' label, for example.
Makes the assembly much cleaner and removes the last
reason anyone would reach for the 123(PC) form instead.
Note that this is on the dev.power64 branch, but it changes all
the assemblers. The change will ship in Go 1.5 (perhaps after
being ported into the new assembler).
Came up as part of CL 167730043.
LGTM=r
R=r
CC=austin, dave, golang-codereviews, minux
https://golang.org/cl/159670043
2014-10-28 19:50:16 -06:00
|
|
|
small:
|
2016-01-15 19:17:09 -07:00
|
|
|
TESTQ BX, BX
|
|
|
|
|
JEQ failure
|
|
|
|
|
|
|
|
|
|
// Check if we'll load across a page boundary.
|
|
|
|
|
LEAQ 16(SI), AX
|
|
|
|
|
TESTW $0xff0, AX
|
|
|
|
|
JEQ endofpage
|
|
|
|
|
|
|
|
|
|
MOVOU (SI), X1 // Load data
|
|
|
|
|
PCMPEQB X0, X1 // Compare target byte with each byte in data.
|
|
|
|
|
PMOVMSKB X1, DX // Move result bits to integer register.
|
|
|
|
|
BSFL DX, DX // Find first set bit.
|
|
|
|
|
JZ failure // No set bit, failure.
|
|
|
|
|
CMPL DX, BX
|
|
|
|
|
JAE failure // Match is past end of data.
|
|
|
|
|
MOVQ DX, (R8)
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
endofpage:
|
|
|
|
|
MOVOU -16(SI)(BX*1), X1 // Load data into the high end of X1.
|
|
|
|
|
PCMPEQB X0, X1 // Compare target byte with each byte in data.
|
|
|
|
|
PMOVMSKB X1, DX // Move result bits to integer register.
|
|
|
|
|
MOVL BX, CX
|
|
|
|
|
SHLL CX, DX
|
|
|
|
|
SHRL $16, DX // Shift desired bits down to bottom of register.
|
|
|
|
|
BSFL DX, DX // Find first set bit.
|
|
|
|
|
JZ failure // No set bit, failure.
|
|
|
|
|
MOVQ DX, (R8)
|
2013-08-01 17:11:19 -06:00
|
|
|
RET
|
|
|
|
|
|
2015-10-29 09:52:22 -06:00
|
|
|
avx2:
|
|
|
|
|
CMPB runtime·support_avx2(SB), $1
|
2016-01-15 19:17:09 -07:00
|
|
|
JNE sse
|
2015-10-29 09:52:22 -06:00
|
|
|
MOVD AX, X0
|
|
|
|
|
LEAQ -32(SI)(BX*1), R11
|
2016-01-22 20:25:15 -07:00
|
|
|
VPBROADCASTB X0, Y1
|
2015-10-29 09:52:22 -06:00
|
|
|
avx2_loop:
|
2016-01-22 20:25:15 -07:00
|
|
|
VMOVDQU (DI), Y2
|
|
|
|
|
VPCMPEQB Y1, Y2, Y3
|
|
|
|
|
VPTEST Y3, Y3
|
2015-10-29 09:52:22 -06:00
|
|
|
JNZ avx2success
|
|
|
|
|
ADDQ $32, DI
|
|
|
|
|
CMPQ DI, R11
|
|
|
|
|
JLT avx2_loop
|
|
|
|
|
MOVQ R11, DI
|
2016-01-22 20:25:15 -07:00
|
|
|
VMOVDQU (DI), Y2
|
|
|
|
|
VPCMPEQB Y1, Y2, Y3
|
|
|
|
|
VPTEST Y3, Y3
|
2015-10-29 09:52:22 -06:00
|
|
|
JNZ avx2success
|
|
|
|
|
VZEROUPPER
|
|
|
|
|
MOVQ $-1, (R8)
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
avx2success:
|
2016-01-22 20:25:15 -07:00
|
|
|
VPMOVMSKB Y3, DX
|
2015-10-29 09:52:22 -06:00
|
|
|
BSFL DX, DX
|
|
|
|
|
SUBQ SI, DI
|
|
|
|
|
ADDQ DI, DX
|
|
|
|
|
MOVQ DX, (R8)
|
|
|
|
|
VZEROUPPER
|
|
|
|
|
RET
|
|
|
|
|
|
2013-08-07 11:23:24 -06:00
|
|
|
TEXT bytes·Equal(SB),NOSPLIT,$0-49
|
2013-08-01 17:11:19 -06:00
|
|
|
MOVQ a_len+8(FP), BX
|
|
|
|
|
MOVQ b_len+32(FP), CX
|
|
|
|
|
CMPQ BX, CX
|
|
|
|
|
JNE eqret
|
|
|
|
|
MOVQ a+0(FP), SI
|
|
|
|
|
MOVQ b+24(FP), DI
|
2015-04-21 15:22:41 -06:00
|
|
|
LEAQ ret+48(FP), AX
|
|
|
|
|
JMP runtime·memeqbody(SB)
|
2013-08-01 17:11:19 -06:00
|
|
|
eqret:
|
2015-04-21 15:22:41 -06:00
|
|
|
MOVB $0, ret+48(FP)
|
2013-08-01 17:11:19 -06:00
|
|
|
RET
|
2014-04-01 13:51:02 -06:00
|
|
|
|
2016-06-28 10:22:46 -06:00
|
|
|
TEXT runtime·fastrand(SB), NOSPLIT, $0-4
|
2014-07-16 15:16:19 -06:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ g(CX), AX
|
|
|
|
|
MOVQ g_m(AX), AX
|
|
|
|
|
MOVL m_fastrand(AX), DX
|
|
|
|
|
ADDL DX, DX
|
|
|
|
|
MOVL DX, BX
|
|
|
|
|
XORL $0x88888eef, DX
|
|
|
|
|
CMOVLMI BX, DX
|
|
|
|
|
MOVL DX, m_fastrand(AX)
|
|
|
|
|
MOVL DX, ret+0(FP)
|
|
|
|
|
RET
|
2014-09-03 09:49:43 -06:00
|
|
|
|
|
|
|
|
TEXT runtime·return0(SB), NOSPLIT, $0
|
|
|
|
|
MOVL $0, AX
|
|
|
|
|
RET
|
2014-09-25 08:59:01 -06:00
|
|
|
|
|
|
|
|
|
|
|
|
|
// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
|
|
|
|
|
// Must obey the gcc calling convention.
|
2014-09-25 09:37:04 -06:00
|
|
|
TEXT _cgo_topofstack(SB),NOSPLIT,$0
|
2014-09-25 08:59:01 -06:00
|
|
|
get_tls(CX)
|
|
|
|
|
MOVQ g(CX), AX
|
|
|
|
|
MOVQ g_m(AX), AX
|
|
|
|
|
MOVQ m_curg(AX), AX
|
|
|
|
|
MOVQ (g_stack+stack_hi)(AX), AX
|
|
|
|
|
RET
|
2014-10-29 18:37:44 -06:00
|
|
|
|
|
|
|
|
// The top-most function running on a goroutine
|
|
|
|
|
// returns to goexit+PCQuantum.
|
|
|
|
|
TEXT runtime·goexit(SB),NOSPLIT,$0-0
|
|
|
|
|
BYTE $0x90 // NOP
|
|
|
|
|
CALL runtime·goexit1(SB) // does not return
|
2015-02-20 10:07:02 -07:00
|
|
|
// traceback from goexit1 must hit code range of goexit
|
|
|
|
|
BYTE $0x90 // NOP
|
[dev.cc] runtime: convert assembly files for C to Go transition
The main change is that #include "zasm_GOOS_GOARCH.h"
is now #include "go_asm.h" and/or #include "go_tls.h".
Also, because C StackGuard is now Go _StackGuard,
the assembly name changes from const_StackGuard to
const__StackGuard.
In asm_$GOARCH.s, add new function getg, formerly
implemented in C.
The renamed atomics now have Go wrappers, to get
escape analysis annotations right. Those wrappers
are in CL 174860043.
LGTM=r, aram
R=r, aram
CC=austin, dvyukov, golang-codereviews, iant, khr
https://golang.org/cl/168510043
2014-11-11 15:06:22 -07:00
|
|
|
|
2014-11-21 13:57:10 -07:00
|
|
|
TEXT runtime·prefetcht0(SB),NOSPLIT,$0-8
|
|
|
|
|
MOVQ addr+0(FP), AX
|
|
|
|
|
PREFETCHT0 (AX)
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
TEXT runtime·prefetcht1(SB),NOSPLIT,$0-8
|
|
|
|
|
MOVQ addr+0(FP), AX
|
|
|
|
|
PREFETCHT1 (AX)
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
TEXT runtime·prefetcht2(SB),NOSPLIT,$0-8
|
|
|
|
|
MOVQ addr+0(FP), AX
|
|
|
|
|
PREFETCHT2 (AX)
|
|
|
|
|
RET
|
|
|
|
|
|
|
|
|
|
TEXT runtime·prefetchnta(SB),NOSPLIT,$0-8
|
|
|
|
|
MOVQ addr+0(FP), AX
|
|
|
|
|
PREFETCHNTA (AX)
|
|
|
|
|
RET
|
2015-03-31 19:17:43 -06:00
|
|
|
|
|
|
|
|
// This is called from .init_array and follows the platform, not Go, ABI.
|
2015-05-11 17:59:14 -06:00
|
|
|
TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
|
|
|
|
|
PUSHQ R15 // The access to global variables below implicitly uses R15, which is callee-save
|
2015-03-31 19:17:43 -06:00
|
|
|
MOVQ runtime·lastmoduledatap(SB), AX
|
|
|
|
|
MOVQ DI, moduledata_next(AX)
|
|
|
|
|
MOVQ DI, runtime·lastmoduledatap(SB)
|
2015-05-11 17:59:14 -06:00
|
|
|
POPQ R15
|
2015-03-31 19:17:43 -06:00
|
|
|
RET
|
