From c20a338c2f354cae51fecec5fa928481b5b3f732 Mon Sep 17 00:00:00 2001 From: Yuval Pavel Zholkover Date: Mon, 29 Aug 2011 10:36:06 -0400 Subject: [PATCH] runtime, syscall: use the vdso page on linux x86 for faster syscalls instead of int $0x80. 8l: fix handling CALL $(constant) code generated by 8a. 8a,8l: add indirect call instruction: CALL *data(SB). R=rsc, iant CC=golang-dev https://golang.org/cl/4817054 --- src/cmd/8a/a.y | 7 +++++ src/cmd/8l/l.h | 1 + src/cmd/8l/optab.c | 3 +- src/cmd/8l/pass.c | 7 +++-- src/cmd/8l/span.c | 14 +++++++++ src/cmd/ld/data.c | 6 +++- src/pkg/runtime/linux/386/rt0.s | 10 ++++++- src/pkg/runtime/linux/386/signal.c | 24 ++++++++++++++++ src/pkg/runtime/linux/386/sys.s | 46 +++++++++++++++++------------- src/pkg/syscall/asm_linux_386.s | 14 ++++----- 10 files changed, 100 insertions(+), 32 deletions(-) diff --git a/src/cmd/8a/a.y b/src/cmd/8a/a.y index a8ac773daa5..96976089def 100644 --- a/src/cmd/8a/a.y +++ b/src/cmd/8a/a.y @@ -210,6 +210,13 @@ spec3: /* JMP/CALL */ $$.from = nullgen; $$.to = $1; } +| '*' nam + { + $$.from = nullgen; + $$.to = $2; + $$.to.index = $2.type; + $$.to.type = D_INDIR+D_ADDR; + } spec4: /* NOP */ nonnon diff --git a/src/cmd/8l/l.h b/src/cmd/8l/l.h index 4ee0db9672c..a721f384b84 100644 --- a/src/cmd/8l/l.h +++ b/src/cmd/8l/l.h @@ -209,6 +209,7 @@ enum Zbr, Zcall, Zcallcon, + Zcallind, Zib_, Zib_rp, Zibo_m, diff --git a/src/cmd/8l/optab.c b/src/cmd/8l/optab.c index 42490b8610c..69602d704a2 100644 --- a/src/cmd/8l/optab.c +++ b/src/cmd/8l/optab.c @@ -260,6 +260,7 @@ uchar yloop[] = uchar ycall[] = { Ynone, Yml, Zo_m, 2, + Ynone, Ycol, Zcallind, 2, Ynone, Ybr, Zcall, 0, Ynone, Yi32, Zcallcon, 1, 0 @@ -383,7 +384,7 @@ Optab optab[] = { ABTSL, yml_rl, Pm, 0xab }, { ABTSW, yml_rl, Pq, 0xab }, { ABYTE, ybyte, Px, 1 }, - { ACALL, ycall, Px, 0xff,(02),0xe8 }, + { ACALL, ycall, Px, 0xff,(02),0xff,(0x15),0xe8 }, { ACLC, ynone, Px, 0xf8 }, { ACLD, ynone, Px, 0xfc }, { ACLI, ynone, Px, 0xfa }, diff --git a/src/cmd/8l/pass.c b/src/cmd/8l/pass.c index 2e0990c5a87..54ea965da2e 100644 --- a/src/cmd/8l/pass.c +++ b/src/cmd/8l/pass.c @@ -307,9 +307,12 @@ patch(void) p->from.offset = 0; } } - if(p->as == ACALL || (p->as == AJMP && p->to.type != D_BRANCH)) { + if((p->as == ACALL && p->to.type != D_BRANCH) || (p->as == AJMP && p->to.type != D_BRANCH)) { s = p->to.sym; - if(s) { + if(p->to.type == D_INDIR+D_ADDR) { + /* skip check if this is an indirect call (CALL *symbol(SB)) */ + continue; + } else if(s) { if(debug['c']) Bprint(&bso, "%s calls %s\n", TNAME, s->name); if((s->type&~SSUB) != STEXT) { diff --git a/src/cmd/8l/span.c b/src/cmd/8l/span.c index ba193da1c61..81c1d37ebbb 100644 --- a/src/cmd/8l/span.c +++ b/src/cmd/8l/span.c @@ -285,6 +285,8 @@ oclass(Adr *a) } return Yxxx; } + //if(a->type == D_INDIR+D_ADDR) + // print("*Ycol\n"); return Ycol; } return Ym; @@ -1143,6 +1145,18 @@ found: r->add = p->to.offset; put4(0); break; + + case Zcallind: + *andptr++ = op; + *andptr++ = o->op[z+1]; + r = addrel(cursym); + r->off = p->pc + andptr - and; + r->type = D_ADDR; + r->siz = 4; + r->add = p->to.offset; + r->sym = p->to.sym; + put4(0); + break; case Zbyte: v = vaddr(&p->from, &rel); diff --git a/src/cmd/ld/data.c b/src/cmd/ld/data.c index e7269169e04..a7f61c927bd 100644 --- a/src/cmd/ld/data.c +++ b/src/cmd/ld/data.c @@ -182,7 +182,11 @@ relocsym(Sym *s) o = symaddr(r->sym) + r->add; break; case D_PCREL: - o = symaddr(r->sym) + r->add - (s->value + r->off + r->siz); + // r->sym can be null when CALL $(constant) is transformed from absoulte PC to relative PC call. + o = 0; + if(r->sym) + o += symaddr(r->sym); + o += r->add - (s->value + r->off + r->siz); break; case D_SIZE: o = r->sym->size + r->add; diff --git a/src/pkg/runtime/linux/386/rt0.s b/src/pkg/runtime/linux/386/rt0.s index 223e6d2ea4f..83149540ec4 100644 --- a/src/pkg/runtime/linux/386/rt0.s +++ b/src/pkg/runtime/linux/386/rt0.s @@ -5,5 +5,13 @@ // Darwin and Linux use the same linkage to main TEXT _rt0_386_linux(SB),7,$0 - JMP _rt0_386(SB) + CALL runtime·linux_setup_vdso(SB) + JMP _rt0_386(SB) + +TEXT _fallback_vdso(SB),7,$0 + INT $0x80 + RET + +DATA runtime·_vdso(SB)/4, $_fallback_vdso(SB) +GLOBL runtime·_vdso(SB), $4 diff --git a/src/pkg/runtime/linux/386/signal.c b/src/pkg/runtime/linux/386/signal.c index 8916e10bd18..4045b2efc3a 100644 --- a/src/pkg/runtime/linux/386/signal.c +++ b/src/pkg/runtime/linux/386/signal.c @@ -182,3 +182,27 @@ os·sigpipe(void) sigaction(SIGPIPE, SIG_DFL, false); runtime·raisesigpipe(); } + +#define AT_NULL 0 +#define AT_SYSINFO 32 +extern uint32 runtime·_vdso; + +#pragma textflag 7 +void runtime·linux_setup_vdso(int32 argc, void *argv_list) +{ + byte **argv = &argv_list; + byte **envp; + uint32 *auxv; + + // skip envp to get to ELF auxiliary vector. + for(envp = &argv[argc+1]; *envp != nil; envp++) + ; + envp++; + + for(auxv=(uint32*)envp; auxv[0] != AT_NULL; auxv += 2) { + if(auxv[0] == AT_SYSINFO) { + runtime·_vdso = auxv[1]; + break; + } + } +} diff --git a/src/pkg/runtime/linux/386/sys.s b/src/pkg/runtime/linux/386/sys.s index 0b4a34986c1..f87420f788c 100644 --- a/src/pkg/runtime/linux/386/sys.s +++ b/src/pkg/runtime/linux/386/sys.s @@ -11,14 +11,14 @@ TEXT runtime·exit(SB),7,$0 MOVL $252, AX // syscall number MOVL 4(SP), BX - INT $0x80 + CALL *runtime·_vdso(SB) INT $3 // not reached RET TEXT runtime·exit1(SB),7,$0 MOVL $1, AX // exit - exit the current os thread MOVL 4(SP), BX - INT $0x80 + CALL *runtime·_vdso(SB) INT $3 // not reached RET @@ -27,13 +27,13 @@ TEXT runtime·open(SB),7,$0 MOVL 4(SP), BX MOVL 8(SP), CX MOVL 12(SP), DX - INT $0x80 + CALL *runtime·_vdso(SB) RET TEXT runtime·close(SB),7,$0 MOVL $6, AX // syscall - close MOVL 4(SP), BX - INT $0x80 + CALL *runtime·_vdso(SB) RET TEXT runtime·write(SB),7,$0 @@ -41,7 +41,7 @@ TEXT runtime·write(SB),7,$0 MOVL 4(SP), BX MOVL 8(SP), CX MOVL 12(SP), DX - INT $0x80 + CALL *runtime·_vdso(SB) RET TEXT runtime·read(SB),7,$0 @@ -49,16 +49,16 @@ TEXT runtime·read(SB),7,$0 MOVL 4(SP), BX MOVL 8(SP), CX MOVL 12(SP), DX - INT $0x80 + CALL *runtime·_vdso(SB) RET TEXT runtime·raisesigpipe(SB),7,$12 MOVL $224, AX // syscall - gettid - INT $0x80 + CALL *runtime·_vdso(SB) MOVL AX, 0(SP) // arg 1 tid MOVL $13, 4(SP) // arg 2 SIGPIPE MOVL $238, AX // syscall - tkill - INT $0x80 + CALL *runtime·_vdso(SB) RET TEXT runtime·setitimer(SB),7,$0-24 @@ -66,7 +66,7 @@ TEXT runtime·setitimer(SB),7,$0-24 MOVL 4(SP), BX MOVL 8(SP), CX MOVL 12(SP), DX - INT $0x80 + CALL *runtime·_vdso(SB) RET TEXT runtime·mincore(SB),7,$0-24 @@ -74,7 +74,7 @@ TEXT runtime·mincore(SB),7,$0-24 MOVL 4(SP), BX MOVL 8(SP), CX MOVL 12(SP), DX - INT $0x80 + CALL *runtime·_vdso(SB) RET TEXT runtime·gettime(SB), 7, $32 @@ -82,7 +82,7 @@ TEXT runtime·gettime(SB), 7, $32 LEAL 8(SP), BX MOVL $0, CX MOVL $0, DX - INT $0x80 + CALL *runtime·_vdso(SB) MOVL 8(SP), BX // sec MOVL sec+0(FP), DI @@ -100,7 +100,7 @@ TEXT runtime·rt_sigaction(SB),7,$0 MOVL 8(SP), CX MOVL 12(SP), DX MOVL 16(SP), SI - INT $0x80 + CALL *runtime·_vdso(SB) RET TEXT runtime·sigtramp(SB),7,$44 @@ -138,7 +138,9 @@ TEXT runtime·sigignore(SB),7,$0 TEXT runtime·sigreturn(SB),7,$0 MOVL $173, AX // rt_sigreturn - INT $0x80 + // Sigreturn expects same SP as signal handler, + // so cannot CALL *runtime._vsdo(SB) here. + INT $0x80 INT $3 // not reached RET @@ -151,7 +153,7 @@ TEXT runtime·mmap(SB),7,$0 MOVL 20(SP), DI MOVL 24(SP), BP SHRL $12, BP - INT $0x80 + CALL *runtime·_vdso(SB) CMPL AX, $0xfffff001 JLS 3(PC) NOTL AX @@ -162,7 +164,7 @@ TEXT runtime·munmap(SB),7,$0 MOVL $91, AX // munmap MOVL 4(SP), BX MOVL 8(SP), CX - INT $0x80 + CALL *runtime·_vdso(SB) CMPL AX, $0xfffff001 JLS 2(PC) INT $3 @@ -178,7 +180,7 @@ TEXT runtime·futex(SB),7,$0 MOVL 16(SP), SI MOVL 20(SP), DI MOVL 24(SP), BP - INT $0x80 + CALL *runtime·_vdso(SB) RET // int32 clone(int32 flags, void *stack, M *m, G *g, void (*fn)(void)); @@ -199,6 +201,10 @@ TEXT runtime·clone(SB),7,$0 MOVL SI, 8(CX) MOVL $1234, 12(CX) + // cannot use CALL *runtime·_vdso(SB) here, because + // the stack changes during the system call (after + // CALL *runtime·_vdso(SB), the child is still using + // the parent's stack when executing its RET instruction). INT $0x80 // In parent, return. @@ -214,7 +220,7 @@ TEXT runtime·clone(SB),7,$0 // Initialize AX to Linux tid MOVL $224, AX - INT $0x80 + CALL *runtime·_vdso(SB) // In child on new stack. Reload registers (paranoia). MOVL 0(SP), BX // m @@ -262,7 +268,7 @@ TEXT runtime·sigaltstack(SB),7,$-8 MOVL $186, AX // sigaltstack MOVL new+4(SP), BX MOVL old+8(SP), CX - INT $0x80 + CALL *runtime·_vdso(SB) CMPL AX, $0xfffff001 JLS 2(PC) INT $3 @@ -323,7 +329,7 @@ TEXT runtime·setldt(SB),7,$32 MOVL AX, CX // user_desc MOVL $16, DX // sizeof(user_desc) MOVL $123, AX // syscall - modify_ldt - INT $0x80 + CALL *runtime·_vdso(SB) // breakpoint on error CMPL AX, $0xfffff001 @@ -340,5 +346,5 @@ TEXT runtime·setldt(SB),7,$32 TEXT runtime·osyield(SB),7,$0 MOVL $158, AX - INT $0x80 + CALL *runtime·_vdso(SB) RET diff --git a/src/pkg/syscall/asm_linux_386.s b/src/pkg/syscall/asm_linux_386.s index 82f170b5bb7..22e00e45b0e 100644 --- a/src/pkg/syscall/asm_linux_386.s +++ b/src/pkg/syscall/asm_linux_386.s @@ -17,7 +17,7 @@ TEXT ·Syscall(SB),7,$0 MOVL 16(SP), DX MOVL $0, SI MOVL $0, DI - INT $0x80 + CALL *runtime·_vdso(SB) CMPL AX, $0xfffff001 JLS ok MOVL $-1, 20(SP) // r1 @@ -43,7 +43,7 @@ TEXT ·Syscall6(SB),7,$0 MOVL 20(SP), SI MOVL 24(SP), DI MOVL 28(SP), BP - INT $0x80 + CALL *runtime·_vdso(SB) CMPL AX, $0xfffff001 JLS ok6 MOVL $-1, 32(SP) // r1 @@ -67,7 +67,7 @@ TEXT ·RawSyscall(SB),7,$0 MOVL 16(SP), DX MOVL $0, SI MOVL $0, DI - INT $0x80 + CALL *runtime·_vdso(SB) CMPL AX, $0xfffff001 JLS ok1 MOVL $-1, 20(SP) // r1 @@ -90,7 +90,7 @@ TEXT ·RawSyscall6(SB),7,$0 MOVL 20(SP), SI MOVL 24(SP), DI MOVL 28(SP), BP - INT $0x80 + CALL *runtime·_vdso(SB) CMPL AX, $0xfffff001 JLS ok2 MOVL $-1, 32(SP) // r1 @@ -116,7 +116,7 @@ TEXT ·socketcall(SB),7,$0 MOVL $0, DX MOVL $0, SI MOVL $0, DI - INT $0x80 + CALL *runtime·_vdso(SB) CMPL AX, $0xfffff001 JLS oksock MOVL $-1, 32(SP) // n @@ -139,7 +139,7 @@ TEXT ·rawsocketcall(SB),7,$0 MOVL $0, DX MOVL $0, SI MOVL $0, DI - INT $0x80 + CALL *runtime·_vdso(SB) CMPL AX, $0xfffff001 JLS oksock1 MOVL $-1, 32(SP) // n @@ -165,7 +165,7 @@ TEXT ·Seek(SB),7,$0 MOVL 8(SP), DX // offset-low LEAL 20(SP), SI // result pointer MOVL 16(SP), DI // whence - INT $0x80 + CALL *runtime·_vdso(SB) CMPL AX, $0xfffff001 JLS okseek MOVL $-1, 20(SP) // newoffset low