[dev.ssa] cmd/compile: reorg write barriers a bit

Use just a single write barrier flag test, even if there are multiple pointer fields in a struct. This helps move more of the wb-specific code (like the LEA needed to materialize the write address) into the unlikely path. Change-Id: Ic7a67145904369c4ff031e464d51267d71281c8f Reviewed-on: https://go-review.googlesource.com/19085 Run-TryBot: Keith Randall <khr@golang.org> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: David Chase <drchase@google.com>
2024-11-24 22:27:57 -07:00 · 2016-01-29 21:57:57 -08:00 · 2016-01-29 21:57:57 -08:00 · aebf6611df
commit aebf6611df
parent a6fb514bf8
2 changed files with 96 additions and 65 deletions
--- a/src/cmd/compile/internal/gc/ssa.go
+++ b/src/cmd/compile/internal/gc/ssa.go
@ -2770,42 +2770,7 @@ func (s *state) insertWBstore(t *Type, left, right *ssa.Value, line int32) {
 	//   store pointer fields
 	// }

-	if t.IsStruct() {
-		n := t.NumFields()
-		for i := int64(0); i < n; i++ {
-			ft := t.FieldType(i)
-			addr := s.newValue1I(ssa.OpOffPtr, ft.PtrTo(), t.FieldOff(i), left)
-			val := s.newValue1I(ssa.OpStructSelect, ft, i, right)
-			if haspointers(ft.(*Type)) {
-				s.insertWBstore(ft.(*Type), addr, val, line)
-			} else {
-				s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, ft.Size(), addr, val, s.mem())
-			}
-		}
-		return
-	}
-
-	switch {
-	case t.IsPtr() || t.IsMap() || t.IsChan():
-		// no scalar fields.
-	case t.IsString():
-		len := s.newValue1(ssa.OpStringLen, Types[TINT], right)
-		lenAddr := s.newValue1I(ssa.OpOffPtr, Ptrto(Types[TINT]), s.config.IntSize, left)
-		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.IntSize, lenAddr, len, s.mem())
-	case t.IsSlice():
-		len := s.newValue1(ssa.OpSliceLen, Types[TINT], right)
-		cap := s.newValue1(ssa.OpSliceCap, Types[TINT], right)
-		lenAddr := s.newValue1I(ssa.OpOffPtr, Ptrto(Types[TINT]), s.config.IntSize, left)
-		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.IntSize, lenAddr, len, s.mem())
-		capAddr := s.newValue1I(ssa.OpOffPtr, Ptrto(Types[TINT]), 2*s.config.IntSize, left)
-		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.IntSize, capAddr, cap, s.mem())
-	case t.IsInterface():
-		// itab field doesn't need a write barrier (even though it is a pointer).
-		itab := s.newValue1(ssa.OpITab, Ptrto(Types[TUINT8]), right)
-		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.IntSize, left, itab, s.mem())
-	default:
-		s.Fatalf("bad write barrier type %s", t)
-	}
+	s.storeTypeScalars(t, left, right)

 	bThen := s.f.NewBlock(ssa.BlockPlain)
 	bElse := s.f.NewBlock(ssa.BlockPlain)
@ -2824,6 +2789,90 @@ func (s *state) insertWBstore(t *Type, left, right *ssa.Value, line int32) {

 	// Issue write barriers for pointer writes.
 	s.startBlock(bThen)
+	s.storeTypePtrsWB(t, left, right)
+	s.endBlock().AddEdgeTo(bEnd)
+
+	// Issue regular stores for pointer writes.
+	s.startBlock(bElse)
+	s.storeTypePtrs(t, left, right)
+	s.endBlock().AddEdgeTo(bEnd)
+
+	s.startBlock(bEnd)
+
+	if Debug_wb > 0 {
+		Warnl(int(line), "write barrier")
+	}
+}
+
+// do *left = right for all scalar (non-pointer) parts of t.
+func (s *state) storeTypeScalars(t *Type, left, right *ssa.Value) {
+	switch {
+	case t.IsBoolean() || t.IsInteger() || t.IsFloat() || t.IsComplex():
+		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, t.Size(), left, right, s.mem())
+	case t.IsPtr() || t.IsMap() || t.IsChan():
+		// no scalar fields.
+	case t.IsString():
+		len := s.newValue1(ssa.OpStringLen, Types[TINT], right)
+		lenAddr := s.newValue1I(ssa.OpOffPtr, Ptrto(Types[TINT]), s.config.IntSize, left)
+		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.IntSize, lenAddr, len, s.mem())
+	case t.IsSlice():
+		len := s.newValue1(ssa.OpSliceLen, Types[TINT], right)
+		cap := s.newValue1(ssa.OpSliceCap, Types[TINT], right)
+		lenAddr := s.newValue1I(ssa.OpOffPtr, Ptrto(Types[TINT]), s.config.IntSize, left)
+		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.IntSize, lenAddr, len, s.mem())
+		capAddr := s.newValue1I(ssa.OpOffPtr, Ptrto(Types[TINT]), 2*s.config.IntSize, left)
+		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.IntSize, capAddr, cap, s.mem())
+	case t.IsInterface():
+		// itab field doesn't need a write barrier (even though it is a pointer).
+		itab := s.newValue1(ssa.OpITab, Ptrto(Types[TUINT8]), right)
+		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.IntSize, left, itab, s.mem())
+	case t.IsStruct():
+		n := t.NumFields()
+		for i := int64(0); i < n; i++ {
+			ft := t.FieldType(i)
+			addr := s.newValue1I(ssa.OpOffPtr, ft.PtrTo(), t.FieldOff(i), left)
+			val := s.newValue1I(ssa.OpStructSelect, ft, i, right)
+			s.storeTypeScalars(ft.(*Type), addr, val)
+		}
+	default:
+		s.Fatalf("bad write barrier type %s", t)
+	}
+}
+
+// do *left = right for all pointer parts of t.
+func (s *state) storeTypePtrs(t *Type, left, right *ssa.Value) {
+	switch {
+	case t.IsPtr() || t.IsMap() || t.IsChan():
+		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.PtrSize, left, right, s.mem())
+	case t.IsString():
+		ptr := s.newValue1(ssa.OpStringPtr, Ptrto(Types[TUINT8]), right)
+		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.PtrSize, left, ptr, s.mem())
+	case t.IsSlice():
+		ptr := s.newValue1(ssa.OpSlicePtr, Ptrto(Types[TUINT8]), right)
+		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.PtrSize, left, ptr, s.mem())
+	case t.IsInterface():
+		// itab field is treated as a scalar.
+		idata := s.newValue1(ssa.OpIData, Ptrto(Types[TUINT8]), right)
+		idataAddr := s.newValue1I(ssa.OpOffPtr, Ptrto(Types[TUINT8]), s.config.PtrSize, left)
+		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.PtrSize, idataAddr, idata, s.mem())
+	case t.IsStruct():
+		n := t.NumFields()
+		for i := int64(0); i < n; i++ {
+			ft := t.FieldType(i)
+			if !haspointers(ft.(*Type)) {
+				continue
+			}
+			addr := s.newValue1I(ssa.OpOffPtr, ft.PtrTo(), t.FieldOff(i), left)
+			val := s.newValue1I(ssa.OpStructSelect, ft, i, right)
+			s.storeTypePtrs(ft.(*Type), addr, val)
+		}
+	default:
+		s.Fatalf("bad write barrier type %s", t)
+	}
+}
+
+// do *left = right with a write barrier for all pointer parts of t.
+func (s *state) storeTypePtrsWB(t *Type, left, right *ssa.Value) {
 	switch {
 	case t.IsPtr() || t.IsMap() || t.IsChan():
 		s.rtcall(writebarrierptr, true, nil, left, right)
@ -2837,36 +2886,20 @@ func (s *state) insertWBstore(t *Type, left, right *ssa.Value, line int32) {
 		idata := s.newValue1(ssa.OpIData, Ptrto(Types[TUINT8]), right)
 		idataAddr := s.newValue1I(ssa.OpOffPtr, Ptrto(Types[TUINT8]), s.config.PtrSize, left)
 		s.rtcall(writebarrierptr, true, nil, idataAddr, idata)
+	case t.IsStruct():
+		n := t.NumFields()
+		for i := int64(0); i < n; i++ {
+			ft := t.FieldType(i)
+			if !haspointers(ft.(*Type)) {
+				continue
+			}
+			addr := s.newValue1I(ssa.OpOffPtr, ft.PtrTo(), t.FieldOff(i), left)
+			val := s.newValue1I(ssa.OpStructSelect, ft, i, right)
+			s.storeTypePtrsWB(ft.(*Type), addr, val)
+		}
 	default:
 		s.Fatalf("bad write barrier type %s", t)
 	}
-	s.endBlock().AddEdgeTo(bEnd)
-
-	// Issue regular stores for pointer writes.
-	s.startBlock(bElse)
-	switch {
-	case t.IsPtr() || t.IsMap() || t.IsChan():
-		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.PtrSize, left, right, s.mem())
-	case t.IsString():
-		ptr := s.newValue1(ssa.OpStringPtr, Ptrto(Types[TUINT8]), right)
-		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.PtrSize, left, ptr, s.mem())
-	case t.IsSlice():
-		ptr := s.newValue1(ssa.OpSlicePtr, Ptrto(Types[TUINT8]), right)
-		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.PtrSize, left, ptr, s.mem())
-	case t.IsInterface():
-		idata := s.newValue1(ssa.OpIData, Ptrto(Types[TUINT8]), right)
-		idataAddr := s.newValue1I(ssa.OpOffPtr, Ptrto(Types[TUINT8]), s.config.PtrSize, left)
-		s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.PtrSize, idataAddr, idata, s.mem())
-	default:
-		s.Fatalf("bad write barrier type %s", t)
-	}
-	s.endBlock().AddEdgeTo(bEnd)
-
-	s.startBlock(bEnd)
-
-	if Debug_wb > 0 {
-		Warnl(int(line), "write barrier")
-	}
 }

 // slice computes the slice v[i:j:k] and returns ptr, len, and cap of result.
--- a/src/cmd/compile/internal/ssa/TODO
+++ b/src/cmd/compile/internal/ssa/TODO
@ -7,7 +7,6 @@ Coverage
 Correctness
 -----------
 - Debugging info (check & fix as much as we can)
- Fix write barriers so cgo tests work (misc/cgo/errors/ptr.go)
 - Re-enable TestStackBarrierProfiling (src/runtime/pprof/pprof_test.go)
 - @ directive in rewrites might read overwritten data.  Save @loc
  in variable before modifying v.
@ -25,7 +24,6 @@ Optimizations (better compiled code)
  SUBQ $8, AX
  CMP AX, $0
  JEQ ...
- Use better write barrier calls
 - If there are a lot of MOVQ $0, ..., then load
  0 into a register and use the register as the source instead.
 - Allow arrays of length 1 (or longer, with all constant indexes?) to be SSAable.