diff --git a/src/cmd/internal/gc/align.go b/src/cmd/internal/gc/align.go index 954e5990211..789e59bfd0a 100644 --- a/src/cmd/internal/gc/align.go +++ b/src/cmd/internal/gc/align.go @@ -411,6 +411,8 @@ func resumecheckwidth() { defercalc = 0 } +var itable *Type // distinguished *byte + func typeinit() { if Widthptr == 0 { Fatal("typeinit before betypeinit") @@ -664,6 +666,9 @@ func typeinit() { dowidth(Types[TSTRING]) dowidth(idealstring) + + itable = typ(Tptr) + itable.Type = Types[TUINT8] } /* diff --git a/src/cmd/internal/gc/cgen.go b/src/cmd/internal/gc/cgen.go index b6691efef53..c996be72fdf 100644 --- a/src/cmd/internal/gc/cgen.go +++ b/src/cmd/internal/gc/cgen.go @@ -13,11 +13,20 @@ import ( * generate: * res = n; * simplifies and calls Thearch.Gmove. + * if wb is true, need to emit write barriers. */ -func Cgen(n *Node, res *Node) { +func Cgen(n, res *Node) { + cgen_wb(n, res, false) +} + +func cgen_wb(n, res *Node, wb bool) { if Debug['g'] != 0 { - Dump("\ncgen-n", n) - Dump("cgen-res", res) + op := "cgen" + if wb { + op = "cgen_wb" + } + Dump("\n"+op+"-n", n) + Dump(op+"-res", res) } if n == nil || n.Type == nil { @@ -34,29 +43,29 @@ func Cgen(n *Node, res *Node) { switch n.Op { case OSLICE, OSLICEARR, OSLICESTR, OSLICE3, OSLICE3ARR: - if res.Op != ONAME || !res.Addable { + if res.Op != ONAME || !res.Addable || wb { var n1 Node Tempname(&n1, n.Type) Cgen_slice(n, &n1) - Cgen(&n1, res) + cgen_wb(&n1, res, wb) } else { Cgen_slice(n, res) } return case OEFACE: - if res.Op != ONAME || !res.Addable { + if res.Op != ONAME || !res.Addable || wb { var n1 Node Tempname(&n1, n.Type) Cgen_eface(n, &n1) - Cgen(&n1, res) + cgen_wb(&n1, res, wb) } else { Cgen_eface(n, res) } return case ODOTTYPE: - cgen_dottype(n, res, nil) + cgen_dottype(n, res, nil, wb) return } @@ -68,7 +77,7 @@ func Cgen(n *Node, res *Node) { var n1 Node Tempname(&n1, n.Type) Cgen(n, &n1) - Cgen(&n1, res) + cgen_wb(&n1, res, wb) return } } @@ -77,7 +86,7 @@ func Cgen(n *Node, res *Node) { if n.Type.Width < 0 { Fatal("forgot to compute width for %v", n.Type) } - sgen(n, res, n.Type.Width) + sgen_wb(n, res, n.Type.Width, wb) return } @@ -87,7 +96,7 @@ func Cgen(n *Node, res *Node) { var n1 Node Tempname(&n1, n.Type) Cgen(n, &n1) - Cgen(&n1, res) + cgen_wb(&n1, res, wb) return } @@ -100,70 +109,67 @@ func Cgen(n *Node, res *Node) { Fatal("loop in cgen") } - Cgen(&n1, res) + cgen_wb(&n1, res, wb) Regfree(&n1) return } var f int - if res.Ullman >= UINF { - goto gen - } + if res.Ullman < UINF { + if Complexop(n, res) { + Complexgen(n, res) + return + } - if Complexop(n, res) { - Complexgen(n, res) - return - } + f = 1 // gen thru register + switch n.Op { + case OLITERAL: + if Smallintconst(n) { + f = 0 + } - f = 1 // gen thru register - switch n.Op { - case OLITERAL: - if Smallintconst(n) { + case OREGISTER: f = 0 } - case OREGISTER: - f = 0 - } - - if !Iscomplex[n.Type.Etype] && Ctxt.Arch.Regsize == 8 { - a := Thearch.Optoas(OAS, res.Type) - var addr obj.Addr - if Thearch.Sudoaddable(a, res, &addr) { - var p1 *obj.Prog - if f != 0 { - var n2 Node - Regalloc(&n2, res.Type, nil) - Cgen(n, &n2) - p1 = Thearch.Gins(a, &n2, nil) - Regfree(&n2) - } else { - p1 = Thearch.Gins(a, n, nil) + if !Iscomplex[n.Type.Etype] && Ctxt.Arch.Regsize == 8 && !wb { + a := Thearch.Optoas(OAS, res.Type) + var addr obj.Addr + if Thearch.Sudoaddable(a, res, &addr) { + var p1 *obj.Prog + if f != 0 { + var n2 Node + Regalloc(&n2, res.Type, nil) + Cgen(n, &n2) + p1 = Thearch.Gins(a, &n2, nil) + Regfree(&n2) + } else { + p1 = Thearch.Gins(a, n, nil) + } + p1.To = addr + if Debug['g'] != 0 { + fmt.Printf("%v [ignore previous line]\n", p1) + } + Thearch.Sudoclean() + return } - p1.To = addr - if Debug['g'] != 0 { - fmt.Printf("%v [ignore previous line]\n", p1) - } - Thearch.Sudoclean() - return } } - gen: if Ctxt.Arch.Thechar == '8' { // no registers to speak of var n1, n2 Node Tempname(&n1, n.Type) Cgen(n, &n1) Igen(res, &n2, nil) - Thearch.Gmove(&n1, &n2) + cgen_wb(&n1, &n2, wb) Regfree(&n2) return } var n1 Node Igen(res, &n1, nil) - Cgen(n, &n1) + cgen_wb(n, &n1, wb) Regfree(&n1) return } @@ -186,6 +192,22 @@ func Cgen(n *Node, res *Node) { n.Addable = n.Left.Addable } + if wb { + if int(Simtype[res.Type.Etype]) != Tptr { + Fatal("cgen_wb of type %v", res.Type) + } + if n.Ullman >= UINF { + var n1 Node + Tempname(&n1, n.Type) + Cgen(n, &n1) + n = &n1 + } + cgen_wbptr(n, res) + return + } + + // Write barrier now handled. Code below this line can ignore wb. + if Ctxt.Arch.Thechar == '5' { // TODO(rsc): Maybe more often? // if both are addressable, move if n.Addable && res.Addable { @@ -765,6 +787,73 @@ abop: // asymmetric binary cgen_norm(n, &n1, res) } +var sys_wbptr *Node + +func cgen_wbptr(n, res *Node) { + if Debug_wb > 0 { + Warn("write barrier") + } + var dst, src Node + Agenr(res, &dst, nil) + Cgenr(n, &src, nil) + p := Thearch.Gins(Thearch.Optoas(OAS, Types[Tptr]), &dst, nil) + a := &p.To + a.Type = obj.TYPE_MEM + a.Reg = int16(Thearch.REGSP) + a.Offset = 0 + if HasLinkRegister() { + a.Offset += int64(Widthptr) + } + p2 := Thearch.Gins(Thearch.Optoas(OAS, Types[Tptr]), &src, nil) + p2.To = p.To + p2.To.Offset += int64(Widthptr) + Regfree(&dst) + Regfree(&src) + if sys_wbptr == nil { + sys_wbptr = writebarrierfn("writebarrierptr", Types[Tptr], Types[Tptr]) + } + Ginscall(sys_wbptr, 0) +} + +func cgen_wbfat(n, res *Node) { + if Debug_wb > 0 { + Warn("write barrier") + } + needType := true + funcName := "typedmemmove" + var dst, src Node + if n.Ullman >= res.Ullman { + Agenr(n, &src, nil) + Agenr(res, &dst, nil) + } else { + Agenr(res, &dst, nil) + Agenr(n, &src, nil) + } + p := Thearch.Gins(Thearch.Optoas(OAS, Types[Tptr]), &dst, nil) + a := &p.To + a.Type = obj.TYPE_MEM + a.Reg = int16(Thearch.REGSP) + a.Offset = 0 + if HasLinkRegister() { + a.Offset += int64(Widthptr) + } + if needType { + a.Offset += int64(Widthptr) + } + p2 := Thearch.Gins(Thearch.Optoas(OAS, Types[Tptr]), &src, nil) + p2.To = p.To + p2.To.Offset += int64(Widthptr) + Regfree(&dst) + Regfree(&src) + if needType { + p3 := Thearch.Gins(Thearch.Optoas(OAS, Types[Tptr]), typename(n.Type), nil) + p3.To = p2.To + p3.To.Offset -= 2 * int64(Widthptr) + Regfree(&src) + } + Ginscall(writebarrierfn(funcName, Types[Tptr], Types[Tptr]), 0) +} + // cgen_norm moves n1 to res, truncating to expected type if necessary. // n1 is a register, and cgen_norm frees it. func cgen_norm(n, n1, res *Node) { @@ -2118,10 +2207,15 @@ func stkof(n *Node) int64 { /* * block copy: * memmove(&ns, &n, w); + * if wb is true, needs write barrier. */ -func sgen(n *Node, ns *Node, w int64) { +func sgen_wb(n *Node, ns *Node, w int64, wb bool) { if Debug['g'] != 0 { - fmt.Printf("\nsgen w=%d\n", w) + op := "sgen" + if wb { + op = "sgen-wb" + } + fmt.Printf("\n%s w=%d\n", op, w) Dump("r", n) Dump("res", ns) } @@ -2145,7 +2239,7 @@ func sgen(n *Node, ns *Node, w int64) { } // Avoid taking the address for simple enough types. - if Componentgen(n, ns) { + if componentgen_wb(n, ns, wb) { return } @@ -2163,15 +2257,29 @@ func sgen(n *Node, ns *Node, w int64) { osrc := stkof(n) odst := stkof(ns) - if osrc != -1000 && odst != -1000 && (osrc == 1000 || odst == 1000) { + if odst != -1000 { + // on stack, write barrier not needed after all + wb = false + } + + if osrc != -1000 && odst != -1000 && (osrc == 1000 || odst == 1000) || wb && osrc != -1000 { // osrc and odst both on stack, and at least one is in // an unknown position. Could generate code to test // for forward/backward copy, but instead just copy // to a temporary location first. + // + // OR: write barrier needed and source is on stack. + // Invoking the write barrier will use the stack to prepare its call. + // Copy to temporary. var tmp Node Tempname(&tmp, n.Type) - sgen(n, &tmp, w) - sgen(&tmp, ns, w) + sgen_wb(n, &tmp, w, false) + sgen_wb(&tmp, ns, w, wb) + return + } + + if wb { + cgen_wbfat(n, ns) return } diff --git a/src/cmd/internal/gc/esc.go b/src/cmd/internal/gc/esc.go index 8d195cdb41a..4a44de7d78a 100644 --- a/src/cmd/internal/gc/esc.go +++ b/src/cmd/internal/gc/esc.go @@ -537,7 +537,7 @@ func esc(e *EscState, n *Node, up *Node) { // This assignment is a no-op for escape analysis, // it does not store any new pointers into b that were not already there. // However, without this special case b will escape, because we assign to OIND/ODOTPTR. - case OAS, OASOP: + case OAS, OASOP, OASWB: if (n.Left.Op == OIND || n.Left.Op == ODOTPTR) && n.Left.Left.Op == ONAME && // dst is ONAME dereference (n.Right.Op == OSLICE || n.Right.Op == OSLICE3 || n.Right.Op == OSLICESTR) && // src is slice operation (n.Right.Left.Op == OIND || n.Right.Left.Op == ODOTPTR) && n.Right.Left.Left.Op == ONAME && // slice is applied to ONAME dereference diff --git a/src/cmd/internal/gc/fmt.go b/src/cmd/internal/gc/fmt.go index 547b873a180..1a991a0a654 100644 --- a/src/cmd/internal/gc/fmt.go +++ b/src/cmd/internal/gc/fmt.go @@ -816,7 +816,7 @@ func stmtfmt(n *Node) string { // Don't export "v = " initializing statements, hope they're always // preceded by the DCL which will be re-parsed and typecheck to reproduce // the "v = " again. - case OAS: + case OAS, OASWB: if fmtmode == FExp && n.Right == nil { break } diff --git a/src/cmd/internal/gc/gen.go b/src/cmd/internal/gc/gen.go index dda33c9dd8d..4c03915c084 100644 --- a/src/cmd/internal/gc/gen.go +++ b/src/cmd/internal/gc/gen.go @@ -403,7 +403,7 @@ func Cgen_eface(n *Node, res *Node) { * n.Left is x * n.Type is T */ -func cgen_dottype(n *Node, res, resok *Node) { +func cgen_dottype(n *Node, res, resok *Node, wb bool) { if Debug_typeassert > 0 { Warn("type assertion inlined") } @@ -441,16 +441,17 @@ func cgen_dottype(n *Node, res, resok *Node) { Cgen(typename(n.Type), &r2) Thearch.Gins(Thearch.Optoas(OCMP, byteptr), &r1, &r2) p := Gbranch(Thearch.Optoas(ONE, byteptr), nil, -1) + Regfree(&r2) // not needed for success path; reclaimed on one failure path iface.Xoffset += int64(Widthptr) Cgen(&iface, &r1) Regfree(&iface) if resok == nil { r1.Type = res.Type - Cgen(&r1, res) + cgen_wb(&r1, res, wb) q := Gbranch(obj.AJMP, nil, 0) Patch(p, Pc) - + Regrealloc(&r2) // reclaim from above, for this failure path fn := syslook("panicdottype", 0) dowidth(fn.Type) call := Nod(OCALLFUNC, fn, nil) @@ -467,10 +468,9 @@ func cgen_dottype(n *Node, res, resok *Node) { // This half is handling the res, resok = x.(T) case, // which is called from gen, not cgen, and is consequently fussier // about blank assignments. We have to avoid calling cgen for those. - Regfree(&r2) r1.Type = res.Type if !isblank(res) { - Cgen(&r1, res) + cgen_wb(&r1, res, wb) } Regfree(&r1) if !isblank(resok) { @@ -979,8 +979,11 @@ func gen(n *Node) { } Cgen_as(n.Left, n.Right) + case OASWB: + Cgen_as_wb(n.Left, n.Right, true) + case OAS2DOTTYPE: - cgen_dottype(n.Rlist.N, n.List.N, n.List.Next.N) + cgen_dottype(n.Rlist.N, n.List.N, n.List.Next.N, false) case OCALLMETH: cgen_callmeth(n, 0) @@ -1023,10 +1026,18 @@ ret: lineno = lno } -func Cgen_as(nl *Node, nr *Node) { +func Cgen_as(nl, nr *Node) { + Cgen_as_wb(nl, nr, false) +} + +func Cgen_as_wb(nl, nr *Node, wb bool) { if Debug['g'] != 0 { - Dump("cgen_as", nl) - Dump("cgen_as = ", nr) + op := "cgen_as" + if wb { + op = "cgen_as_wb" + } + Dump(op, nl) + Dump(op+" = ", nr) } for nr != nil && nr.Op == OCONVNOP { @@ -1065,7 +1076,7 @@ func Cgen_as(nl *Node, nr *Node) { return } - Cgen(nr, nl) + cgen_wb(nr, nl, wb) } func cgen_callmeth(n *Node, proc int) { @@ -1126,31 +1137,40 @@ func checklabels() { // Slices, strings and interfaces are supported. Small structs or arrays with // elements of basic type are also supported. // nr is nil when assigning a zero value. -func Componentgen(nr *Node, nl *Node) bool { +func Componentgen(nr, nl *Node) bool { + return componentgen_wb(nr, nl, false) +} + +// componentgen_wb is like componentgen but if wb==true emits write barriers for pointer updates. +func componentgen_wb(nr, nl *Node, wb bool) bool { + // Don't generate any code for complete copy of a variable into itself. + // It's useless, and the VARDEF will incorrectly mark the old value as dead. + // (This check assumes that the arguments passed to componentgen did not + // themselves come from Igen, or else we could have Op==ONAME but + // with a Type and Xoffset describing an individual field, not the entire + // variable.) + if nl.Op == ONAME && nl == nr { + return true + } + // Count number of moves required to move components. + // If using write barrier, can only emit one pointer. + // TODO(rsc): Allow more pointers, for reflect.Value. const maxMoves = 8 n := 0 + numPtr := 0 visitComponents(nl.Type, 0, func(t *Type, offset int64) bool { n++ - return n <= maxMoves + if int(Simtype[t.Etype]) == Tptr && t != itable { + numPtr++ + } + return n <= maxMoves && (!wb || numPtr <= 1) }) - if n > maxMoves { + if n > maxMoves || wb && numPtr > 1 { return false } - isConstString := Isconst(nr, CTSTR) - nodl := *nl - if !cadable(nl) { - if nr != nil && !cadable(nr) && !isConstString { - return false - } - Igen(nl, &nodl, nil) - defer Regfree(&nodl) - } - lbase := nodl.Xoffset - - // Must call emitVardef on every path out of this function, - // but only after evaluating rhs. + // Must call emitVardef after evaluating rhs but before writing to lhs. emitVardef := func() { // Emit vardef if needed. if nl.Op == ONAME { @@ -1161,6 +1181,26 @@ func Componentgen(nr *Node, nl *Node) bool { } } + isConstString := Isconst(nr, CTSTR) + + if !cadable(nl) && nr != nil && !cadable(nr) && !isConstString { + return false + } + + var nodl Node + if cadable(nl) { + nodl = *nl + } else { + if nr != nil && !cadable(nr) && !isConstString { + return false + } + if nr == nil || isConstString || nl.Ullman >= nr.Ullman { + Igen(nl, &nodl, nil) + defer Regfree(&nodl) + } + } + lbase := nodl.Xoffset + // Special case: zeroing. var nodr Node if nr == nil { @@ -1218,23 +1258,34 @@ func Componentgen(nr *Node, nl *Node) bool { // General case: copy nl = nr. nodr = *nr if !cadable(nr) { + if nr.Ullman >= UINF && nodl.Op == OINDREG { + Fatal("miscompile") + } Igen(nr, &nodr, nil) defer Regfree(&nodr) } rbase := nodr.Xoffset - // Don't generate any code for complete copy of a variable into itself. - // It's useless, and the VARDEF will incorrectly mark the old value as dead. - // (This check assumes that the arguments passed to componentgen did not - // themselves come from Igen, or else we could have Op==ONAME but - // with a Type and Xoffset describing an individual field, not the entire - // variable.) - if nl.Op == ONAME && nr.Op == ONAME && nl == nr { - return true + if nodl.Op == 0 { + Igen(nl, &nodl, nil) + defer Regfree(&nodl) + lbase = nodl.Xoffset } emitVardef() + var ( + ptrType *Type + ptrOffset int64 + ) visitComponents(nl.Type, 0, func(t *Type, offset int64) bool { + if wb && int(Simtype[t.Etype]) == Tptr && t != itable { + if ptrType != nil { + Fatal("componentgen_wb %v", Tconv(nl.Type, 0)) + } + ptrType = t + ptrOffset = offset + return true + } nodl.Type = t nodl.Xoffset = lbase + offset nodr.Type = t @@ -1242,6 +1293,13 @@ func Componentgen(nr *Node, nl *Node) bool { Thearch.Gmove(&nodr, &nodl) return true }) + if ptrType != nil { + nodl.Type = ptrType + nodl.Xoffset = lbase + ptrOffset + nodr.Type = ptrType + nodr.Xoffset = rbase + ptrOffset + cgen_wbptr(&nodr, &nodl) + } return true } @@ -1283,7 +1341,7 @@ func visitComponents(t *Type, startOffset int64, f func(elem *Type, elemOffset i f(Types[TFLOAT64], startOffset+8) case TINTER: - return f(Ptrto(Types[TUINT8]), startOffset) && + return f(itable, startOffset) && f(Ptrto(Types[TUINT8]), startOffset+int64(Widthptr)) return true diff --git a/src/cmd/internal/gc/init.go b/src/cmd/internal/gc/init.go index b3a6a00d00b..b5d1e505a57 100644 --- a/src/cmd/internal/gc/init.go +++ b/src/cmd/internal/gc/init.go @@ -57,7 +57,7 @@ func anyinit(n *NodeList) bool { case ODCLFUNC, ODCLCONST, ODCLTYPE, OEMPTY: break - case OAS: + case OAS, OASWB: if isblank(l.N.Left) && candiscard(l.N.Right) { break } diff --git a/src/cmd/internal/gc/racewalk.go b/src/cmd/internal/gc/racewalk.go index 934cfe7a198..e7f35006dc6 100644 --- a/src/cmd/internal/gc/racewalk.go +++ b/src/cmd/internal/gc/racewalk.go @@ -137,7 +137,7 @@ func racewalknode(np **Node, init **NodeList, wr int, skip int) { default: Fatal("racewalk: unknown node type %v", Oconv(int(n.Op), 0)) - case OAS, OAS2FUNC: + case OAS, OASWB, OAS2FUNC: racewalknode(&n.Left, init, 1, 0) racewalknode(&n.Right, init, 0, 0) goto ret diff --git a/src/cmd/internal/gc/subr.go b/src/cmd/internal/gc/subr.go index cfe1b879ef5..381079e40c5 100644 --- a/src/cmd/internal/gc/subr.go +++ b/src/cmd/internal/gc/subr.go @@ -1614,7 +1614,7 @@ func ullmancalc(n *Node) { } goto out - case OCALL, OCALLFUNC, OCALLMETH, OCALLINTER: + case OCALL, OCALLFUNC, OCALLMETH, OCALLINTER, OASWB: ul = UINF goto out diff --git a/src/cmd/internal/gc/syntax.go b/src/cmd/internal/gc/syntax.go index d448188a550..e9593fdcb96 100644 --- a/src/cmd/internal/gc/syntax.go +++ b/src/cmd/internal/gc/syntax.go @@ -170,6 +170,7 @@ const ( OAS2MAPR // x, ok = m["foo"] OAS2DOTTYPE // x, ok = I.(int) OASOP // x += y + OASWB // OAS but with write barrier OCALL // function call, method call or type conversion, possibly preceded by defer or go. OCALLFUNC // f() OCALLMETH // t.Method() diff --git a/src/cmd/internal/gc/walk.go b/src/cmd/internal/gc/walk.go index 043edc9f4f8..79a303855a2 100644 --- a/src/cmd/internal/gc/walk.go +++ b/src/cmd/internal/gc/walk.go @@ -2218,10 +2218,14 @@ func applywritebarrier(n *Node, init **NodeList) *Node { if Curfn != nil && Curfn.Func.Nowritebarrier { Yyerror("write barrier prohibited") } + t := n.Left.Type + if t.Width == int64(Widthptr) { + n.Op = OASWB + return n + } if Debug_wb > 0 { Warnl(int(n.Lineno), "write barrier") } - t := n.Left.Type l := Nod(OADDR, n.Left, nil) l.Etype = 1 // addr does not escape if t.Width == int64(Widthptr) { @@ -2274,7 +2278,6 @@ func applywritebarrier(n *Node, init **NodeList) *Node { n = mkcall1(writebarrierfn("typedmemmove", t, r.Left.Type), nil, init, typename(t), l, r) } } - return n }