[dev.ssa] cmd/compile/internal/ssa: enforce load-store ordering in scheduler

We must make sure that all loads that use a store are scheduled
before the next store.  Add additional dependency edges to the
value graph to enforce this constraint.

Change-Id: Iab83644f68bc4c30637085b82ca7467b9d5513a5
Reviewed-on: https://go-review.googlesource.com/13470
Reviewed-by: Josh Bleecher Snyder <josharian@gmail.com>

src/cmd/compile/internal/gc/testdata/loadstore_ssa.go

@@ -0,0 +1,39 @@
+// run
+
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Tests load/store ordering
+
+package main
+
+// testLoadStoreOrder tests for reordering of stores/loads.
+func testLoadStoreOrder() {
+	z := uint32(1000)
+	if testLoadStoreOrder_ssa(&z, 100) == 0 {
+		println("testLoadStoreOrder failed")
+		failed = true
+	}
+}
+func testLoadStoreOrder_ssa(z *uint32, prec uint) int {
+	switch {
+	}
+	old := *z         // load
+	*z = uint32(prec) // store
+	if *z < old {     // load
+		return 1
+	}
+	return 0
+}
+
+var failed = false
+
+func main() {
+
+	testLoadStoreOrder()
+
+	if failed {
+		panic("failed")
+	}
+}

src/cmd/compile/internal/ssa/schedule.go

@@ -21,22 +21,49 @@ func schedule(f *Func) {
 	var order []*Value
 
 	// priority queue of legally schedulable (0 unscheduled uses) values
-	var priq [4][]*Value
+	var priq [5][]*Value
+
+	// maps mem values to the next live memory value
+	nextMem := make([]*Value, f.NumValues())
+	// additional pretend arguments for each Value.  Used to enforce load/store ordering.
+	additionalArgs := make([][]*Value, f.NumValues())
 
 	for _, b := range f.Blocks {
-		// Compute uses.
+		// Find store chain for block.
 		for _, v := range b.Values {
-			if v.Op != OpPhi {
-				// Note: if a value is used by a phi, it does not induce
-				// a scheduling edge because that use is from the
-				// previous iteration.
+			if v.Op != OpPhi && v.Type.IsMemory() {
 				for _, w := range v.Args {
-					if w.Block == b {
-						uses[w.ID]++
+					if w.Type.IsMemory() {
+						nextMem[w.ID] = v
 					}
 				}
 			}
 		}
+
+		// Compute uses.
+		for _, v := range b.Values {
+			if v.Op == OpPhi {
+				// If a value is used by a phi, it does not induce
+				// a scheduling edge because that use is from the
+				// previous iteration.
+				continue
+			}
+			for _, w := range v.Args {
+				if w.Block == b {
+					uses[w.ID]++
+				}
+				// Any load must come before the following store.
+				if v.Type.IsMemory() || !w.Type.IsMemory() {
+					continue // not a load
+				}
+				s := nextMem[w.ID]
+				if s == nil || s.Block != b {
+					continue
+				}
+				additionalArgs[s.ID] = append(additionalArgs[s.ID], v)
+				uses[v.ID]++
+			}
+		}
 		// Compute score.  Larger numbers are scheduled closer to the end of the block.
 		for _, v := range b.Values {
 			switch {
@@ -44,23 +71,22 @@ func schedule(f *Func) {
 				// We want all the phis first.
 				score[v.ID] = 0
 			case v.Type.IsMemory():
-				// Schedule stores as late as possible.
-				// This makes sure that loads do not get scheduled
-				// after a following store (1-live-memory requirement).
-				score[v.ID] = 2
+				// Schedule stores as early as possible.  This tends to
+				// reduce register pressure.
+				score[v.ID] = 1
 			case v.Type.IsFlags():
 				// Schedule flag register generation as late as possible.
 				// This makes sure that we only have one live flags
 				// value at a time.
-				score[v.ID] = 2
+				score[v.ID] = 3
 			default:
-				score[v.ID] = 1
+				score[v.ID] = 2
 			}
 		}
 		if b.Control != nil && b.Control.Op != OpPhi {
 			// Force the control value to be scheduled at the end,
 			// unless it is a phi value (which must be first).
-			score[b.Control.ID] = 3
+			score[b.Control.ID] = 4
 			// TODO: some times control values are used by other values
 			// in the block.  So the control value will not appear at
 			// the very end.  Decide if this is a problem or not.
@@ -110,6 +136,14 @@ func schedule(f *Func) {
 					priq[s] = append(priq[s], w)
 				}
 			}
+			for _, w := range additionalArgs[v.ID] {
+				uses[w.ID]--
+				if uses[w.ID] == 0 {
+					// All uses scheduled, w is now schedulable.
+					s := score[w.ID]
+					priq[s] = append(priq[s], w)
+				}
+			}
 		}
 		if len(order) != len(b.Values) {
 			f.Fatalf("schedule does not include all values")