cmd/compile: use desired register only if it satisfies register mask

In the register allocator, if possible, we allocate a value to its
desired register (the ideal register for its next use). In some
cases the desired register does not satisfies the value's output
register mask. We should not use the register in this case.

In the following example, v33 is going to be returned as a
function result, so it is allocated to its desired register AX.
However, its Op cannot use AX as output, causing miscompilation.

v33 = CMOVQEQF <int> v24 v28 v29 : AX (~R0[int])
v35 = MakeResult <int,int,mem> v33 v26 v18
Ret v35

Change-Id: Id0f4f27c4b233ee297e83077e3c8494fe193e664
Reviewed-on: https://go-review.googlesource.com/c/go/+/314630
Trust: Cherry Zhang <cherryyz@google.com>
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Than McIntosh <thanm@google.com>

src/cmd/compile/internal/ssa/regalloc.go

@@ -1493,9 +1493,9 @@ func (s *regAllocState) regalloc(f *Func) {
 					goto ok
 				}
 
-				// Try to move an input to the desired output.
+				// Try to move an input to the desired output, if allowed.
 				for _, r := range dinfo[idx].out {
-					if r != noRegister && m>>r&1 != 0 {
+					if r != noRegister && (m&regspec.outputs[0].regs)>>r&1 != 0 {
 						m = regMask(1) << r
 						args[0] = s.allocValToReg(v.Args[0], m, true, v.Pos)
 						// Note: we update args[0] so the instruction will
@@ -1569,6 +1569,9 @@ func (s *regAllocState) regalloc(f *Func) {
 						if !opcodeTable[v.Op].commutative {
 							// Output must use the same register as input 0.
 							r := register(s.f.getHome(args[0].ID).(*Register).num)
+							if mask>>r&1 == 0 {
+								s.f.Fatalf("resultInArg0 value's input %v cannot be an output of %s", s.f.getHome(args[0].ID).(*Register), v.LongString())
+							}
 							mask = regMask(1) << r
 						} else {
 							// Output must use the same register as input 0 or 1.

test/abi/result_regalloc.go

@@ -0,0 +1,46 @@
+// run
+
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Bug: in (*bb).d, the value to be returned was not allocated to
+// a register that satisfies its register mask.
+
+package main
+
+type bb struct {
+	r float64
+	x []float64
+}
+
+//go:noinline
+func B(r float64, x []float64) I {
+	return bb{r, x}
+}
+
+func (b bb) d() (int, int) {
+	if b.r == 0 {
+		return 0, len(b.x)
+	}
+	return len(b.x), len(b.x)
+}
+
+type I interface { d() (int, int) }
+
+func D(r I) (int, int) { return r.d() }
+
+//go:noinline
+func F() (int, int) {
+	r := float64(1)
+	x := []float64{0, 1, 2}
+	b := B(r, x)
+	return D(b)
+}
+
+func main() {
+	x, y := F()
+	if x != 3 || y != 3 {
+		panic("FAIL")
+	}
+}