[dev.ssa] cmd/compile/internal/ssa: fix shift operations

Convert shift ops to also encode the size of the shift amount.

Change signed right shift from using CMOV to using bit twiddles.
It is a little bit better (5 instructions instead of 4, but fewer
bytes and slightly faster code).  It's also a bit faster than
the 4-instruction branch version, even with a very predictable
branch.  As tested on my machine, YMMV.

Implement OCOM while we are here.

Change-Id: I8ca12dd62fae5d626dc0e6da5d4bbd34fd9640d2
Reviewed-on: https://go-review.googlesource.com/12867
Reviewed-by: Josh Bleecher Snyder <josharian@gmail.com>

src/cmd/compile/internal/gc/ssa.go

@@ -727,6 +727,15 @@ var opToSSA = map[opAndType]ssa.Op{
 	opAndType{OMINUS, TINT64}:  ssa.OpNeg64,
 	opAndType{OMINUS, TUINT64}: ssa.OpNeg64,
 
+	opAndType{OCOM, TINT8}:   ssa.OpCom8,
+	opAndType{OCOM, TUINT8}:  ssa.OpCom8,
+	opAndType{OCOM, TINT16}:  ssa.OpCom16,
+	opAndType{OCOM, TUINT16}: ssa.OpCom16,
+	opAndType{OCOM, TINT32}:  ssa.OpCom32,
+	opAndType{OCOM, TUINT32}: ssa.OpCom32,
+	opAndType{OCOM, TINT64}:  ssa.OpCom64,
+	opAndType{OCOM, TUINT64}: ssa.OpCom64,
+
 	opAndType{OMUL, TINT8}:   ssa.OpMul8,
 	opAndType{OMUL, TUINT8}:  ssa.OpMul8,
 	opAndType{OMUL, TINT16}:  ssa.OpMul16,
@@ -754,24 +763,6 @@ var opToSSA = map[opAndType]ssa.Op{
 	opAndType{OOR, TINT64}:  ssa.OpOr64,
 	opAndType{OOR, TUINT64}: ssa.OpOr64,
 
-	opAndType{OLSH, TINT8}:   ssa.OpLsh8,
-	opAndType{OLSH, TUINT8}:  ssa.OpLsh8,
-	opAndType{OLSH, TINT16}:  ssa.OpLsh16,
-	opAndType{OLSH, TUINT16}: ssa.OpLsh16,
-	opAndType{OLSH, TINT32}:  ssa.OpLsh32,
-	opAndType{OLSH, TUINT32}: ssa.OpLsh32,
-	opAndType{OLSH, TINT64}:  ssa.OpLsh64,
-	opAndType{OLSH, TUINT64}: ssa.OpLsh64,
-
-	opAndType{ORSH, TINT8}:   ssa.OpRsh8,
-	opAndType{ORSH, TUINT8}:  ssa.OpRsh8U,
-	opAndType{ORSH, TINT16}:  ssa.OpRsh16,
-	opAndType{ORSH, TUINT16}: ssa.OpRsh16U,
-	opAndType{ORSH, TINT32}:  ssa.OpRsh32,
-	opAndType{ORSH, TUINT32}: ssa.OpRsh32U,
-	opAndType{ORSH, TINT64}:  ssa.OpRsh64,
-	opAndType{ORSH, TUINT64}: ssa.OpRsh64U,
-
 	opAndType{OEQ, TBOOL}:      ssa.OpEq8,
 	opAndType{OEQ, TINT8}:      ssa.OpEq8,
 	opAndType{OEQ, TUINT8}:     ssa.OpEq8,
@@ -877,6 +868,96 @@ func (s *state) ssaOp(op uint8, t *Type) ssa.Op {
 	return x
 }
 
+type opAndTwoTypes struct {
+	op     uint8
+	etype1 uint8
+	etype2 uint8
+}
+
+var shiftOpToSSA = map[opAndTwoTypes]ssa.Op{
+	opAndTwoTypes{OLSH, TINT8, TUINT8}:   ssa.OpLsh8x8,
+	opAndTwoTypes{OLSH, TUINT8, TUINT8}:  ssa.OpLsh8x8,
+	opAndTwoTypes{OLSH, TINT8, TUINT16}:  ssa.OpLsh8x16,
+	opAndTwoTypes{OLSH, TUINT8, TUINT16}: ssa.OpLsh8x16,
+	opAndTwoTypes{OLSH, TINT8, TUINT32}:  ssa.OpLsh8x32,
+	opAndTwoTypes{OLSH, TUINT8, TUINT32}: ssa.OpLsh8x32,
+	opAndTwoTypes{OLSH, TINT8, TUINT64}:  ssa.OpLsh8x64,
+	opAndTwoTypes{OLSH, TUINT8, TUINT64}: ssa.OpLsh8x64,
+
+	opAndTwoTypes{OLSH, TINT16, TUINT8}:   ssa.OpLsh16x8,
+	opAndTwoTypes{OLSH, TUINT16, TUINT8}:  ssa.OpLsh16x8,
+	opAndTwoTypes{OLSH, TINT16, TUINT16}:  ssa.OpLsh16x16,
+	opAndTwoTypes{OLSH, TUINT16, TUINT16}: ssa.OpLsh16x16,
+	opAndTwoTypes{OLSH, TINT16, TUINT32}:  ssa.OpLsh16x32,
+	opAndTwoTypes{OLSH, TUINT16, TUINT32}: ssa.OpLsh16x32,
+	opAndTwoTypes{OLSH, TINT16, TUINT64}:  ssa.OpLsh16x64,
+	opAndTwoTypes{OLSH, TUINT16, TUINT64}: ssa.OpLsh16x64,
+
+	opAndTwoTypes{OLSH, TINT32, TUINT8}:   ssa.OpLsh32x8,
+	opAndTwoTypes{OLSH, TUINT32, TUINT8}:  ssa.OpLsh32x8,
+	opAndTwoTypes{OLSH, TINT32, TUINT16}:  ssa.OpLsh32x16,
+	opAndTwoTypes{OLSH, TUINT32, TUINT16}: ssa.OpLsh32x16,
+	opAndTwoTypes{OLSH, TINT32, TUINT32}:  ssa.OpLsh32x32,
+	opAndTwoTypes{OLSH, TUINT32, TUINT32}: ssa.OpLsh32x32,
+	opAndTwoTypes{OLSH, TINT32, TUINT64}:  ssa.OpLsh32x64,
+	opAndTwoTypes{OLSH, TUINT32, TUINT64}: ssa.OpLsh32x64,
+
+	opAndTwoTypes{OLSH, TINT64, TUINT8}:   ssa.OpLsh64x8,
+	opAndTwoTypes{OLSH, TUINT64, TUINT8}:  ssa.OpLsh64x8,
+	opAndTwoTypes{OLSH, TINT64, TUINT16}:  ssa.OpLsh64x16,
+	opAndTwoTypes{OLSH, TUINT64, TUINT16}: ssa.OpLsh64x16,
+	opAndTwoTypes{OLSH, TINT64, TUINT32}:  ssa.OpLsh64x32,
+	opAndTwoTypes{OLSH, TUINT64, TUINT32}: ssa.OpLsh64x32,
+	opAndTwoTypes{OLSH, TINT64, TUINT64}:  ssa.OpLsh64x64,
+	opAndTwoTypes{OLSH, TUINT64, TUINT64}: ssa.OpLsh64x64,
+
+	opAndTwoTypes{ORSH, TINT8, TUINT8}:   ssa.OpRsh8x8,
+	opAndTwoTypes{ORSH, TUINT8, TUINT8}:  ssa.OpRsh8Ux8,
+	opAndTwoTypes{ORSH, TINT8, TUINT16}:  ssa.OpRsh8x16,
+	opAndTwoTypes{ORSH, TUINT8, TUINT16}: ssa.OpRsh8Ux16,
+	opAndTwoTypes{ORSH, TINT8, TUINT32}:  ssa.OpRsh8x32,
+	opAndTwoTypes{ORSH, TUINT8, TUINT32}: ssa.OpRsh8Ux32,
+	opAndTwoTypes{ORSH, TINT8, TUINT64}:  ssa.OpRsh8x64,
+	opAndTwoTypes{ORSH, TUINT8, TUINT64}: ssa.OpRsh8Ux64,
+
+	opAndTwoTypes{ORSH, TINT16, TUINT8}:   ssa.OpRsh16x8,
+	opAndTwoTypes{ORSH, TUINT16, TUINT8}:  ssa.OpRsh16Ux8,
+	opAndTwoTypes{ORSH, TINT16, TUINT16}:  ssa.OpRsh16x16,
+	opAndTwoTypes{ORSH, TUINT16, TUINT16}: ssa.OpRsh16Ux16,
+	opAndTwoTypes{ORSH, TINT16, TUINT32}:  ssa.OpRsh16x32,
+	opAndTwoTypes{ORSH, TUINT16, TUINT32}: ssa.OpRsh16Ux32,
+	opAndTwoTypes{ORSH, TINT16, TUINT64}:  ssa.OpRsh16x64,
+	opAndTwoTypes{ORSH, TUINT16, TUINT64}: ssa.OpRsh16Ux64,
+
+	opAndTwoTypes{ORSH, TINT32, TUINT8}:   ssa.OpRsh32x8,
+	opAndTwoTypes{ORSH, TUINT32, TUINT8}:  ssa.OpRsh32Ux8,
+	opAndTwoTypes{ORSH, TINT32, TUINT16}:  ssa.OpRsh32x16,
+	opAndTwoTypes{ORSH, TUINT32, TUINT16}: ssa.OpRsh32Ux16,
+	opAndTwoTypes{ORSH, TINT32, TUINT32}:  ssa.OpRsh32x32,
+	opAndTwoTypes{ORSH, TUINT32, TUINT32}: ssa.OpRsh32Ux32,
+	opAndTwoTypes{ORSH, TINT32, TUINT64}:  ssa.OpRsh32x64,
+	opAndTwoTypes{ORSH, TUINT32, TUINT64}: ssa.OpRsh32Ux64,
+
+	opAndTwoTypes{ORSH, TINT64, TUINT8}:   ssa.OpRsh64x8,
+	opAndTwoTypes{ORSH, TUINT64, TUINT8}:  ssa.OpRsh64Ux8,
+	opAndTwoTypes{ORSH, TINT64, TUINT16}:  ssa.OpRsh64x16,
+	opAndTwoTypes{ORSH, TUINT64, TUINT16}: ssa.OpRsh64Ux16,
+	opAndTwoTypes{ORSH, TINT64, TUINT32}:  ssa.OpRsh64x32,
+	opAndTwoTypes{ORSH, TUINT64, TUINT32}: ssa.OpRsh64Ux32,
+	opAndTwoTypes{ORSH, TINT64, TUINT64}:  ssa.OpRsh64x64,
+	opAndTwoTypes{ORSH, TUINT64, TUINT64}: ssa.OpRsh64Ux64,
+}
+
+func (s *state) ssaShiftOp(op uint8, t *Type, u *Type) ssa.Op {
+	etype1 := s.concreteEtype(t)
+	etype2 := s.concreteEtype(u)
+	x, ok := shiftOpToSSA[opAndTwoTypes{op, etype1, etype2}]
+	if !ok {
+		s.Unimplementedf("unhandled shift op %s etype=%s/%s", opnames[op], Econv(int(etype1), 0), Econv(int(etype2), 0))
+	}
+	return x
+}
+
 // expr converts the expression n to ssa, adds it to s and returns the ssa result.
 func (s *state) expr(n *Node) *ssa.Value {
 	s.pushLine(n.Lineno)
@@ -999,10 +1080,14 @@ func (s *state) expr(n *Node) *ssa.Value {
 		a := s.expr(n.Left)
 		b := s.expr(n.Right)
 		return s.newValue2(s.ssaOp(n.Op, n.Left.Type), ssa.TypeBool, a, b)
-	case OADD, OAND, OLSH, OMUL, OOR, ORSH, OSUB:
+	case OADD, OSUB, OMUL, OAND, OOR:
 		a := s.expr(n.Left)
 		b := s.expr(n.Right)
 		return s.newValue2(s.ssaOp(n.Op, n.Type), a.Type, a, b)
+	case OLSH, ORSH:
+		a := s.expr(n.Left)
+		b := s.expr(n.Right)
+		return s.newValue2(s.ssaShiftOp(n.Op, n.Type, n.Right.Type), a.Type, a, b)
 	case OANDAND, OOROR:
 		// To implement OANDAND (and OOROR), we introduce a
 		// new temporary variable to hold the result. The
@@ -1045,7 +1130,7 @@ func (s *state) expr(n *Node) *ssa.Value {
 		return s.variable(n, n.Type)
 
 	// unary ops
-	case ONOT, OMINUS:
+	case ONOT, OMINUS, OCOM:
 		a := s.expr(n.Left)
 		return s.newValue1(s.ssaOp(n.Op, n.Type), a.Type, a)
 
@@ -1766,36 +1851,13 @@ func genValue(v *ssa.Value) {
 		p.From.Offset = v.AuxInt
 		p.To.Type = obj.TYPE_REG
 		p.To.Reg = r
-	case ssa.OpAMD64SBBQcarrymask:
+	case ssa.OpAMD64SBBQcarrymask, ssa.OpAMD64SBBLcarrymask:
 		r := regnum(v)
 		p := Prog(v.Op.Asm())
 		p.From.Type = obj.TYPE_REG
 		p.From.Reg = r
 		p.To.Type = obj.TYPE_REG
 		p.To.Reg = r
-	case ssa.OpAMD64CMOVQCC:
-		r := regnum(v)
-		x := regnum(v.Args[1])
-		y := regnum(v.Args[2])
-		if x != r && y != r {
-			p := Prog(x86.AMOVQ)
-			p.From.Type = obj.TYPE_REG
-			p.From.Reg = x
-			p.To.Type = obj.TYPE_REG
-			p.To.Reg = r
-			x = r
-		}
-		var p *obj.Prog
-		if x == r {
-			p = Prog(x86.ACMOVQCS)
-			p.From.Reg = y
-		} else {
-			p = Prog(x86.ACMOVQCC)
-			p.From.Reg = x
-		}
-		p.From.Type = obj.TYPE_REG
-		p.To.Type = obj.TYPE_REG
-		p.To.Reg = r
 	case ssa.OpAMD64LEAQ1, ssa.OpAMD64LEAQ2, ssa.OpAMD64LEAQ4, ssa.OpAMD64LEAQ8:
 		p := Prog(x86.ALEAQ)
 		p.From.Type = obj.TYPE_MEM
@@ -1967,7 +2029,8 @@ func genValue(v *ssa.Value) {
 		p := Prog(obj.ACALL)
 		p.To.Type = obj.TYPE_REG
 		p.To.Reg = regnum(v.Args[0])
-	case ssa.OpAMD64NEGQ, ssa.OpAMD64NEGL, ssa.OpAMD64NEGW, ssa.OpAMD64NEGB:
+	case ssa.OpAMD64NEGQ, ssa.OpAMD64NEGL, ssa.OpAMD64NEGW, ssa.OpAMD64NEGB,
+		ssa.OpAMD64NOTQ, ssa.OpAMD64NOTL, ssa.OpAMD64NOTW, ssa.OpAMD64NOTB:
 		p := Prog(v.Op.Asm())
 		p.To.Type = obj.TYPE_REG
 		p.To.Reg = regnum(v.Args[0])

src/cmd/compile/internal/ssa/gen/AMD64.rules

@@ -50,6 +50,11 @@
 (Neg16 x) -> (NEGW x)
 (Neg8 x) -> (NEGB x)
 
+(Com64 x) -> (NOTQ x)
+(Com32 x) -> (NOTL x)
+(Com16 x) -> (NOTW x)
+(Com8 x) -> (NOTB x)
+
 // Note: we always extend to 64 bits even though some ops don't need that many result bits.
 (SignExt8to16 x) -> (MOVBQSX x)
 (SignExt8to32 x) -> (MOVBQSX x)
@@ -78,57 +83,71 @@
 // TODO: other ConvNops are safe?  Maybe all of them?
 
 // Lowering shifts
-// Note: unsigned shifts need to return 0 if shift amount is >= 64.
+// Unsigned shifts need to return 0 if shift amount is >= width of shifted value.
-//   mask = shift >= 64 ? 0 : 0xffffffffffffffff
+//   result = (arg << shift) & (shift >= argbits ? 0 : 0xffffffffffffffff)
-//   result = mask & arg << shift
+// Note: for small shifts we generate 32 bits of mask even when we don't need it all.
-// TODO: define ops per right-hand side size, like Lsh64x32 for int64(x)<<uint32(y)?
+(Lsh64x64 <t> x y) -> (ANDQ (SHLQ <t> x y) (SBBQcarrymask <t> (CMPQconst <TypeFlags> [64] y)))
-(Lsh64 <t> x y) && y.Type.Size() == 8 ->
+(Lsh64x32 <t> x y) -> (ANDQ (SHLQ <t> x y) (SBBQcarrymask <t> (CMPLconst <TypeFlags> [64] y)))
-	(ANDQ (SHLQ <t> x y) (SBBQcarrymask <t> (CMPQconst <TypeFlags> [64] y)))
+(Lsh64x16 <t> x y) -> (ANDQ (SHLQ <t> x y) (SBBQcarrymask <t> (CMPWconst <TypeFlags> [64] y)))
-(Lsh64 <t> x y) && y.Type.Size() == 4 ->
+(Lsh64x8 <t> x y)  -> (ANDQ (SHLQ <t> x y) (SBBQcarrymask <t> (CMPBconst <TypeFlags> [64] y)))
-	(ANDQ (SHLQ <t> x y) (SBBQcarrymask <t> (CMPLconst <TypeFlags> [64] y)))
-(Lsh64 <t> x y) && y.Type.Size() == 2 ->
-	(ANDQ (SHLQ <t> x y) (SBBQcarrymask <t> (CMPWconst <TypeFlags> [64] y)))
-(Lsh64 <t> x y) && y.Type.Size() == 1 ->
-	(ANDQ (SHLQ <t> x y) (SBBQcarrymask <t> (CMPBconst <TypeFlags> [64] y)))
 
-(Lsh32 <t> x y) && y.Type.Size() == 8 ->
+(Lsh32x64 <t> x y) -> (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMPQconst <TypeFlags> [32] y)))
-	(ANDL (SHLL <t> x y) (SBBQcarrymask <t> (CMPQconst <TypeFlags> [32] y)))
+(Lsh32x32 <t> x y) -> (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMPLconst <TypeFlags> [32] y)))
-(Lsh32 <t> x y) && y.Type.Size() == 4 ->
+(Lsh32x16 <t> x y) -> (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMPWconst <TypeFlags> [32] y)))
-	(ANDL (SHLL <t> x y) (SBBQcarrymask <t> (CMPLconst <TypeFlags> [32] y)))
+(Lsh32x8 <t> x y)  -> (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMPBconst <TypeFlags> [32] y)))
-(Lsh32 <t> x y) && y.Type.Size() == 2 ->
-	(ANDL (SHLL <t> x y) (SBBQcarrymask <t> (CMPWconst <TypeFlags> [32] y)))
-(Lsh32 <t> x y) && y.Type.Size() == 1 ->
-	(ANDL (SHLL <t> x y) (SBBQcarrymask <t> (CMPBconst <TypeFlags> [32] y)))
 
-(Lsh16 <t> x y) && y.Type.Size() == 8 ->
+(Lsh16x64 <t> x y) -> (ANDW (SHLW <t> x y) (SBBLcarrymask <t> (CMPQconst <TypeFlags> [16] y)))
-	(ANDW (SHLW <t> x y) (SBBQcarrymask <t> (CMPQconst <TypeFlags> [16] y)))
+(Lsh16x32 <t> x y) -> (ANDW (SHLW <t> x y) (SBBLcarrymask <t> (CMPLconst <TypeFlags> [16] y)))
-(Lsh16 <t> x y) && y.Type.Size() == 4 ->
+(Lsh16x16 <t> x y) -> (ANDW (SHLW <t> x y) (SBBLcarrymask <t> (CMPWconst <TypeFlags> [16] y)))
-	(ANDW (SHLW <t> x y) (SBBQcarrymask <t> (CMPLconst <TypeFlags> [16] y)))
+(Lsh16x8 <t> x y)  -> (ANDW (SHLW <t> x y) (SBBLcarrymask <t> (CMPBconst <TypeFlags> [16] y)))
-(Lsh16 <t> x y) && y.Type.Size() == 2 ->
-	(ANDW (SHLW <t> x y) (SBBQcarrymask <t> (CMPWconst <TypeFlags> [16] y)))
-(Lsh16 <t> x y) && y.Type.Size() == 1 ->
-	(ANDW (SHLW <t> x y) (SBBQcarrymask <t> (CMPBconst <TypeFlags> [16] y)))
 
-(Lsh8 <t> x y) && y.Type.Size() == 8 ->
+(Lsh8x64 <t> x y)  -> (ANDB (SHLB <t> x y) (SBBLcarrymask <t> (CMPQconst <TypeFlags> [8] y)))
-	(ANDB (SHLB <t> x y) (SBBQcarrymask <t> (CMPQconst <TypeFlags> [8] y)))
+(Lsh8x32 <t> x y)  -> (ANDB (SHLB <t> x y) (SBBLcarrymask <t> (CMPLconst <TypeFlags> [8] y)))
-(Lsh8 <t> x y) && y.Type.Size() == 4 ->
+(Lsh8x16 <t> x y)  -> (ANDB (SHLB <t> x y) (SBBLcarrymask <t> (CMPWconst <TypeFlags> [8] y)))
-	(ANDB (SHLB <t> x y) (SBBQcarrymask <t> (CMPLconst <TypeFlags> [8] y)))
+(Lsh8x8 <t> x y)   -> (ANDB (SHLB <t> x y) (SBBLcarrymask <t> (CMPBconst <TypeFlags> [8] y)))
-(Lsh8 <t> x y) && y.Type.Size() == 2 ->
-	(ANDB (SHLB <t> x y) (SBBQcarrymask <t> (CMPWconst <TypeFlags> [8] y)))
-(Lsh8 <t> x y) && y.Type.Size() == 1 ->
-	(ANDB (SHLB <t> x y) (SBBQcarrymask <t> (CMPBconst <TypeFlags> [8] y)))
 
-(Rsh64U <t> x y) && y.Type.Size() == 8 ->
+(Rsh64Ux64 <t> x y) -> (ANDQ (SHRQ <t> x y) (SBBQcarrymask <t> (CMPQconst <TypeFlags> [64] y)))
-	(ANDQ (SHRQ <t> x y) (SBBQcarrymask <t> (CMPQconst <TypeFlags> [64] y)))
+(Rsh64Ux32 <t> x y) -> (ANDQ (SHRQ <t> x y) (SBBQcarrymask <t> (CMPLconst <TypeFlags> [64] y)))
+(Rsh64Ux16 <t> x y) -> (ANDQ (SHRQ <t> x y) (SBBQcarrymask <t> (CMPWconst <TypeFlags> [64] y)))
+(Rsh64Ux8 <t> x y)  -> (ANDQ (SHRQ <t> x y) (SBBQcarrymask <t> (CMPBconst <TypeFlags> [64] y)))
 
-// Note: signed right shift needs to return 0/-1 if shift amount is >= 64.
+(Rsh32Ux64 <t> x y) -> (ANDL (SHRL <t> x y) (SBBLcarrymask <t> (CMPQconst <TypeFlags> [32] y)))
-//   if shift > 63 { shift = 63 }
+(Rsh32Ux32 <t> x y) -> (ANDL (SHRL <t> x y) (SBBLcarrymask <t> (CMPLconst <TypeFlags> [32] y)))
-//   result = arg >> shift
+(Rsh32Ux16 <t> x y) -> (ANDL (SHRL <t> x y) (SBBLcarrymask <t> (CMPWconst <TypeFlags> [32] y)))
-(Rsh64 <t> x y) && y.Type.Size() == 8 ->
+(Rsh32Ux8 <t> x y)  -> (ANDL (SHRL <t> x y) (SBBLcarrymask <t> (CMPBconst <TypeFlags> [32] y)))
-	(SARQ <t> x (CMOVQCC <t>
+
-			(CMPQconst <TypeFlags> [64] y)
+(Rsh16Ux64 <t> x y) -> (ANDW (SHRW <t> x y) (SBBLcarrymask <t> (CMPQconst <TypeFlags> [16] y)))
-			(MOVQconst <t> [63])
+(Rsh16Ux32 <t> x y) -> (ANDW (SHRW <t> x y) (SBBLcarrymask <t> (CMPLconst <TypeFlags> [16] y)))
-			y))
+(Rsh16Ux16 <t> x y) -> (ANDW (SHRW <t> x y) (SBBLcarrymask <t> (CMPWconst <TypeFlags> [16] y)))
+(Rsh16Ux8 <t> x y)  -> (ANDW (SHRW <t> x y) (SBBLcarrymask <t> (CMPBconst <TypeFlags> [16] y)))
+
+(Rsh8Ux64 <t> x y)  -> (ANDB (SHRB <t> x y) (SBBLcarrymask <t> (CMPQconst <TypeFlags> [8] y)))
+(Rsh8Ux32 <t> x y)  -> (ANDB (SHRB <t> x y) (SBBLcarrymask <t> (CMPLconst <TypeFlags> [8] y)))
+(Rsh8Ux16 <t> x y)  -> (ANDB (SHRB <t> x y) (SBBLcarrymask <t> (CMPWconst <TypeFlags> [8] y)))
+(Rsh8Ux8 <t> x y)   -> (ANDB (SHRB <t> x y) (SBBLcarrymask <t> (CMPBconst <TypeFlags> [8] y)))
+
+// Signed right shift needs to return 0/-1 if shift amount is >= width of shifted value.
+// We implement this by setting the shift value to -1 (all ones) if the shift value is >= width.
+// Note: for small shift widths we generate 32 bits of mask even when we don't need it all.
+(Rsh64x64 <t> x y) -> (SARQ <t> x (ORQ <y.Type> y (NOTQ <y.Type> (SBBQcarrymask <y.Type> (CMPQconst <TypeFlags> [64] y)))))
+(Rsh64x32 <t> x y) -> (SARQ <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPLconst <TypeFlags> [64] y)))))
+(Rsh64x16 <t> x y) -> (SARQ <t> x (ORW <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPWconst <TypeFlags> [64] y)))))
+(Rsh64x8 <t> x y)  -> (SARQ <t> x (ORB <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPBconst <TypeFlags> [64] y)))))
+
+(Rsh32x64 <t> x y) -> (SARL <t> x (ORQ <y.Type> y (NOTQ <y.Type> (SBBQcarrymask <y.Type> (CMPQconst <TypeFlags> [32] y)))))
+(Rsh32x32 <t> x y) -> (SARL <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPLconst <TypeFlags> [32] y)))))
+(Rsh32x16 <t> x y) -> (SARL <t> x (ORW <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPWconst <TypeFlags> [32] y)))))
+(Rsh32x8 <t> x y)  -> (SARL <t> x (ORB <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPBconst <TypeFlags> [32] y)))))
+
+(Rsh16x64 <t> x y) -> (SARW <t> x (ORQ <y.Type> y (NOTQ <y.Type> (SBBQcarrymask <y.Type> (CMPQconst <TypeFlags> [16] y)))))
+(Rsh16x32 <t> x y) -> (SARW <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPLconst <TypeFlags> [16] y)))))
+(Rsh16x16 <t> x y) -> (SARW <t> x (ORW <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPWconst <TypeFlags> [16] y)))))
+(Rsh16x8 <t> x y)  -> (SARW <t> x (ORB <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPBconst <TypeFlags> [16] y)))))
+
+(Rsh8x64 <t> x y)  -> (SARB <t> x (ORQ <y.Type> y (NOTQ <y.Type> (SBBQcarrymask <y.Type> (CMPQconst <TypeFlags> [8] y)))))
+(Rsh8x32 <t> x y)  -> (SARB <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPLconst <TypeFlags> [8] y)))))
+(Rsh8x16 <t> x y)  -> (SARB <t> x (ORW <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPWconst <TypeFlags> [8] y)))))
+(Rsh8x8 <t> x y)   -> (SARB <t> x (ORB <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPBconst <TypeFlags> [8] y)))))
 
 (Less64 x y) -> (SETL (CMPQ <TypeFlags> x y))
 (Less32 x y) -> (SETL (CMPL <TypeFlags> x y))
@@ -398,10 +417,58 @@
 (EQ (InvertFlags cmp) yes no) -> (EQ cmp yes no)
 (NE (InvertFlags cmp) yes no) -> (NE cmp yes no)
 
-// get rid of >=64 code for constant shifts
+// get rid of overflow code for constant shifts
 (SBBQcarrymask (CMPQconst [c] (MOVQconst [d]))) && inBounds(d, c) -> (MOVQconst [-1])
 (SBBQcarrymask (CMPQconst [c] (MOVQconst [d]))) && !inBounds(d, c) -> (MOVQconst [0])
-(ANDQconst [0] _) -> (MOVQconst [0])
+(SBBQcarrymask (CMPLconst [c] (MOVLconst [d]))) && inBounds(int64(int32(d)), int64(int32(c))) -> (MOVQconst [-1])
-(ANDQconst [-1] x) -> (Copy x)
+(SBBQcarrymask (CMPLconst [c] (MOVLconst [d]))) && !inBounds(int64(int32(d)), int64(int32(c))) -> (MOVQconst [0])
-(CMOVQCC (CMPQconst [c] (MOVQconst [d])) _ x) && inBounds(d, c) -> (Copy x)
+(SBBQcarrymask (CMPWconst [c] (MOVWconst [d]))) && inBounds(int64(int16(d)), int64(int16(c))) -> (MOVQconst [-1])
-(CMOVQCC (CMPQconst [c] (MOVQconst [d])) x _) && !inBounds(d, c) -> (Copy x)
+(SBBQcarrymask (CMPWconst [c] (MOVWconst [d]))) && !inBounds(int64(int16(d)), int64(int16(c))) -> (MOVQconst [0])
+(SBBQcarrymask (CMPBconst [c] (MOVBconst [d]))) && inBounds(int64(int8(d)), int64(int8(c))) -> (MOVQconst [-1])
+(SBBQcarrymask (CMPBconst [c] (MOVBconst [d]))) && !inBounds(int64(int8(d)), int64(int8(c))) -> (MOVQconst [0])
+(ANDQconst [0] _)                 -> (MOVQconst [0])
+(ANDLconst [c] _) && int32(c)==0  -> (MOVLconst [0])
+(ANDWconst [c] _) && int16(c)==0  -> (MOVWconst [0])
+(ANDBconst [c] _) && int8(c)==0   -> (MOVBconst [0])
+(ANDQconst [-1] x)                -> (Copy x)
+(ANDLconst [c] x) && int32(c)==-1 -> (Copy x)
+(ANDWconst [c] x) && int16(c)==-1 -> (Copy x)
+(ANDBconst [c] x) && int8(c)==-1  -> (Copy x)
+(ORQconst [0] x)                  -> (Copy x)
+(ORLconst [c] x) && int32(c)==0   -> (Copy x)
+(ORWconst [c] x) && int16(c)==0   -> (Copy x)
+(ORBconst [c] x) && int8(c)==0    -> (Copy x)
+(ORQconst [-1] _)                 -> (MOVQconst [-1])
+(ORLconst [c] _) && int32(c)==-1  -> (MOVLconst [-1])
+(ORWconst [c] _) && int16(c)==-1  -> (MOVWconst [-1])
+(ORBconst [c] _) && int8(c)==-1   -> (MOVBconst [-1])
+
+// generic constant folding
+// TODO: more of this
+(ADDQconst [c] (MOVQconst [d])) -> (MOVQconst [c+d])
+(ADDLconst [c] (MOVLconst [d])) -> (MOVLconst [c+d])
+(ADDWconst [c] (MOVWconst [d])) -> (MOVWconst [c+d])
+(ADDBconst [c] (MOVBconst [d])) -> (MOVBconst [c+d])
+(SUBQconst [c] (MOVQconst [d])) -> (MOVQconst [c-d])
+(SUBLconst [c] (MOVLconst [d])) -> (MOVLconst [c-d])
+(SUBWconst [c] (MOVWconst [d])) -> (MOVWconst [c-d])
+(SUBBconst [c] (MOVBconst [d])) -> (MOVBconst [c-d])
+(MULQconst [c] (MOVQconst [d])) -> (MOVQconst [c*d])
+(MULLconst [c] (MOVLconst [d])) -> (MOVLconst [c*d])
+(MULWconst [c] (MOVWconst [d])) -> (MOVWconst [c*d])
+(ANDQconst [c] (MOVQconst [d])) -> (MOVQconst [c&d])
+(ANDLconst [c] (MOVLconst [d])) -> (MOVLconst [c&d])
+(ANDWconst [c] (MOVWconst [d])) -> (MOVWconst [c&d])
+(ANDBconst [c] (MOVBconst [d])) -> (MOVBconst [c&d])
+(ORQconst [c] (MOVQconst [d])) -> (MOVQconst [c|d])
+(ORLconst [c] (MOVLconst [d])) -> (MOVLconst [c|d])
+(ORWconst [c] (MOVWconst [d])) -> (MOVWconst [c|d])
+(ORBconst [c] (MOVBconst [d])) -> (MOVBconst [c|d])
+(XORQconst [c] (MOVQconst [d])) -> (MOVQconst [c^d])
+(XORLconst [c] (MOVLconst [d])) -> (MOVLconst [c^d])
+(XORWconst [c] (MOVWconst [d])) -> (MOVWconst [c^d])
+(XORBconst [c] (MOVBconst [d])) -> (MOVBconst [c^d])
+(NOTQ (MOVQconst [c])) -> (MOVQconst [^c])
+(NOTL (MOVLconst [c])) -> (MOVLconst [^c])
+(NOTW (MOVWconst [c])) -> (MOVWconst [^c])
+(NOTB (MOVBconst [c])) -> (MOVBconst [^c])

src/cmd/compile/internal/ssa/gen/AMD64Ops.go

@@ -89,7 +89,6 @@ func init() {
 	gpstoreconst := regInfo{[]regMask{gpspsb, 0}, 0, nil}
 	gpstoreidx := regInfo{[]regMask{gpspsb, gpsp, gpsp, 0}, 0, nil}
 	flagsgp := regInfo{[]regMask{flags}, 0, []regMask{gp}}
-	cmov := regInfo{[]regMask{flags, gp, gp}, 0, []regMask{gp}}
 
 	// Suffixes encode the bit width of various instructions.
 	// Q = 64 bit, L = 32 bit, W = 16 bit, B = 8 bit
@@ -201,7 +200,14 @@ func init() {
 		{name: "NEGW", reg: gp11, asm: "NEGW"}, // -arg0
 		{name: "NEGB", reg: gp11, asm: "NEGB"}, // -arg0
 
+		{name: "NOTQ", reg: gp11, asm: "NOTQ"}, // ^arg0
+		{name: "NOTL", reg: gp11, asm: "NOTL"}, // ^arg0
+		{name: "NOTW", reg: gp11, asm: "NOTW"}, // ^arg0
+		{name: "NOTB", reg: gp11, asm: "NOTB"}, // ^arg0
+
 		{name: "SBBQcarrymask", reg: flagsgp1, asm: "SBBQ"}, // (int64)(-1) if carry is set, 0 if carry is clear.
+		{name: "SBBLcarrymask", reg: flagsgp1, asm: "SBBL"}, // (int32)(-1) if carry is set, 0 if carry is clear.
+		// Note: SBBW and SBBB are subsumed by SBBL
 
 		{name: "SETEQ", reg: flagsgp, asm: "SETEQ"}, // extract == condition from arg0
 		{name: "SETNE", reg: flagsgp, asm: "SETNE"}, // extract != condition from arg0
@@ -214,8 +220,6 @@ func init() {
 		{name: "SETA", reg: flagsgp, asm: "SETHI"},  // extract unsigned > condition from arg0
 		{name: "SETAE", reg: flagsgp, asm: "SETCC"}, // extract unsigned >= condition from arg0
 
-		{name: "CMOVQCC", reg: cmov}, // carry clear
-
 		{name: "MOVBQSX", reg: gp11, asm: "MOVBQSX"}, // sign extend arg0 from int8 to int64
 		{name: "MOVBQZX", reg: gp11, asm: "MOVBQZX"}, // zero extend arg0 from int8 to int64
 		{name: "MOVWQSX", reg: gp11, asm: "MOVWQSX"}, // sign extend arg0 from int16 to int64

src/cmd/compile/internal/ssa/gen/genericOps.go

@@ -42,19 +42,57 @@ var genericOps = []opData{
 	{name: "Xor32"},
 	{name: "Xor64"},
 
-	{name: "Lsh8"}, // arg0 << arg1
+	// For shifts, AxB means the shifted value has A bits and the shift amount has B bits.
-	{name: "Lsh16"},
+	{name: "Lsh8x8"}, // arg0 << arg1
-	{name: "Lsh32"},
+	{name: "Lsh8x16"},
-	{name: "Lsh64"},
+	{name: "Lsh8x32"},
+	{name: "Lsh8x64"},
+	{name: "Lsh16x8"},
+	{name: "Lsh16x16"},
+	{name: "Lsh16x32"},
+	{name: "Lsh16x64"},
+	{name: "Lsh32x8"},
+	{name: "Lsh32x16"},
+	{name: "Lsh32x32"},
+	{name: "Lsh32x64"},
+	{name: "Lsh64x8"},
+	{name: "Lsh64x16"},
+	{name: "Lsh64x32"},
+	{name: "Lsh64x64"},
 
-	{name: "Rsh8"}, // arg0 >> arg1
+	{name: "Rsh8x8"}, // arg0 >> arg1, signed
-	{name: "Rsh8U"},
+	{name: "Rsh8x16"},
-	{name: "Rsh16"},
+	{name: "Rsh8x32"},
-	{name: "Rsh16U"},
+	{name: "Rsh8x64"},
-	{name: "Rsh32"},
+	{name: "Rsh16x8"},
-	{name: "Rsh32U"},
+	{name: "Rsh16x16"},
-	{name: "Rsh64"},
+	{name: "Rsh16x32"},
-	{name: "Rsh64U"},
+	{name: "Rsh16x64"},
+	{name: "Rsh32x8"},
+	{name: "Rsh32x16"},
+	{name: "Rsh32x32"},
+	{name: "Rsh32x64"},
+	{name: "Rsh64x8"},
+	{name: "Rsh64x16"},
+	{name: "Rsh64x32"},
+	{name: "Rsh64x64"},
+
+	{name: "Rsh8Ux8"}, // arg0 >> arg1, unsigned
+	{name: "Rsh8Ux16"},
+	{name: "Rsh8Ux32"},
+	{name: "Rsh8Ux64"},
+	{name: "Rsh16Ux8"},
+	{name: "Rsh16Ux16"},
+	{name: "Rsh16Ux32"},
+	{name: "Rsh16Ux64"},
+	{name: "Rsh32Ux8"},
+	{name: "Rsh32Ux16"},
+	{name: "Rsh32Ux32"},
+	{name: "Rsh32Ux64"},
+	{name: "Rsh64Ux8"},
+	{name: "Rsh64Ux16"},
+	{name: "Rsh64Ux32"},
+	{name: "Rsh64Ux64"},
 
 	// 2-input comparisons
 	{name: "Eq8"}, // arg0 == arg1
@@ -110,11 +148,16 @@ var genericOps = []opData{
 	// 1-input ops
 	{name: "Not"}, // !arg0
 
-	{name: "Neg8"}, // - arg0
+	{name: "Neg8"}, // -arg0
 	{name: "Neg16"},
 	{name: "Neg32"},
 	{name: "Neg64"},
 
+	{name: "Com8"}, // ^arg0
+	{name: "Com16"},
+	{name: "Com32"},
+	{name: "Com64"},
+
 	// Data movement
 	{name: "Phi"},  // select an argument based on which predecessor block we came from
 	{name: "Copy"}, // output = arg0

src/cmd/compile/internal/ssa/opGen.go

@@ -141,7 +141,12 @@ const (
 	OpAMD64NEGL
 	OpAMD64NEGW
 	OpAMD64NEGB
+	OpAMD64NOTQ
+	OpAMD64NOTL
+	OpAMD64NOTW
+	OpAMD64NOTB
 	OpAMD64SBBQcarrymask
+	OpAMD64SBBLcarrymask
 	OpAMD64SETEQ
 	OpAMD64SETNE
 	OpAMD64SETL
@@ -152,7 +157,6 @@ const (
 	OpAMD64SETBE
 	OpAMD64SETA
 	OpAMD64SETAE
-	OpAMD64CMOVQCC
 	OpAMD64MOVBQSX
 	OpAMD64MOVBQZX
 	OpAMD64MOVWQSX
@@ -213,18 +217,54 @@ const (
 	OpXor16
 	OpXor32
 	OpXor64
-	OpLsh8
+	OpLsh8x8
-	OpLsh16
+	OpLsh8x16
-	OpLsh32
+	OpLsh8x32
-	OpLsh64
+	OpLsh8x64
-	OpRsh8
+	OpLsh16x8
-	OpRsh8U
+	OpLsh16x16
-	OpRsh16
+	OpLsh16x32
-	OpRsh16U
+	OpLsh16x64
-	OpRsh32
+	OpLsh32x8
-	OpRsh32U
+	OpLsh32x16
-	OpRsh64
+	OpLsh32x32
-	OpRsh64U
+	OpLsh32x64
+	OpLsh64x8
+	OpLsh64x16
+	OpLsh64x32
+	OpLsh64x64
+	OpRsh8x8
+	OpRsh8x16
+	OpRsh8x32
+	OpRsh8x64
+	OpRsh16x8
+	OpRsh16x16
+	OpRsh16x32
+	OpRsh16x64
+	OpRsh32x8
+	OpRsh32x16
+	OpRsh32x32
+	OpRsh32x64
+	OpRsh64x8
+	OpRsh64x16
+	OpRsh64x32
+	OpRsh64x64
+	OpRsh8Ux8
+	OpRsh8Ux16
+	OpRsh8Ux32
+	OpRsh8Ux64
+	OpRsh16Ux8
+	OpRsh16Ux16
+	OpRsh16Ux32
+	OpRsh16Ux64
+	OpRsh32Ux8
+	OpRsh32Ux16
+	OpRsh32Ux32
+	OpRsh32Ux64
+	OpRsh64Ux8
+	OpRsh64Ux16
+	OpRsh64Ux32
+	OpRsh64Ux64
 	OpEq8
 	OpEq16
 	OpEq32
@@ -274,6 +314,10 @@ const (
 	OpNeg16
 	OpNeg32
 	OpNeg64
+	OpCom8
+	OpCom16
+	OpCom32
+	OpCom64
 	OpPhi
 	OpCopy
 	OpConstBool
@@ -1458,6 +1502,54 @@ var opcodeTable = [...]opInfo{
 			},
 		},
 	},
+	{
+		name: "NOTQ",
+		asm:  x86.ANOTQ,
+		reg: regInfo{
+			inputs: []regMask{
+				65535, // .AX .CX .DX .BX .SP .BP .SI .DI .R8 .R9 .R10 .R11 .R12 .R13 .R14 .R15
+			},
+			outputs: []regMask{
+				65519, // .AX .CX .DX .BX .BP .SI .DI .R8 .R9 .R10 .R11 .R12 .R13 .R14 .R15
+			},
+		},
+	},
+	{
+		name: "NOTL",
+		asm:  x86.ANOTL,
+		reg: regInfo{
+			inputs: []regMask{
+				65535, // .AX .CX .DX .BX .SP .BP .SI .DI .R8 .R9 .R10 .R11 .R12 .R13 .R14 .R15
+			},
+			outputs: []regMask{
+				65519, // .AX .CX .DX .BX .BP .SI .DI .R8 .R9 .R10 .R11 .R12 .R13 .R14 .R15
+			},
+		},
+	},
+	{
+		name: "NOTW",
+		asm:  x86.ANOTW,
+		reg: regInfo{
+			inputs: []regMask{
+				65535, // .AX .CX .DX .BX .SP .BP .SI .DI .R8 .R9 .R10 .R11 .R12 .R13 .R14 .R15
+			},
+			outputs: []regMask{
+				65519, // .AX .CX .DX .BX .BP .SI .DI .R8 .R9 .R10 .R11 .R12 .R13 .R14 .R15
+			},
+		},
+	},
+	{
+		name: "NOTB",
+		asm:  x86.ANOTB,
+		reg: regInfo{
+			inputs: []regMask{
+				65535, // .AX .CX .DX .BX .SP .BP .SI .DI .R8 .R9 .R10 .R11 .R12 .R13 .R14 .R15
+			},
+			outputs: []regMask{
+				65519, // .AX .CX .DX .BX .BP .SI .DI .R8 .R9 .R10 .R11 .R12 .R13 .R14 .R15
+			},
+		},
+	},
 	{
 		name: "SBBQcarrymask",
 		asm:  x86.ASBBQ,
@@ -1470,6 +1562,18 @@ var opcodeTable = [...]opInfo{
 			},
 		},
 	},
+	{
+		name: "SBBLcarrymask",
+		asm:  x86.ASBBL,
+		reg: regInfo{
+			inputs: []regMask{
+				8589934592, // .FLAGS
+			},
+			outputs: []regMask{
+				65519, // .AX .CX .DX .BX .BP .SI .DI .R8 .R9 .R10 .R11 .R12 .R13 .R14 .R15
+			},
+		},
+	},
 	{
 		name: "SETEQ",
 		asm:  x86.ASETEQ,
@@ -1590,19 +1694,6 @@ var opcodeTable = [...]opInfo{
 			},
 		},
 	},
-	{
-		name: "CMOVQCC",
-		reg: regInfo{
-			inputs: []regMask{
-				8589934592, // .FLAGS
-				65519,      // .AX .CX .DX .BX .BP .SI .DI .R8 .R9 .R10 .R11 .R12 .R13 .R14 .R15
-				65519,      // .AX .CX .DX .BX .BP .SI .DI .R8 .R9 .R10 .R11 .R12 .R13 .R14 .R15
-			},
-			outputs: []regMask{
-				65519, // .AX .CX .DX .BX .BP .SI .DI .R8 .R9 .R10 .R11 .R12 .R13 .R14 .R15
-			},
-		},
-	},
 	{
 		name: "MOVBQSX",
 		asm:  x86.AMOVBQSX,
@@ -2072,51 +2163,195 @@ var opcodeTable = [...]opInfo{
 		generic: true,
 	},
 	{
-		name:    "Lsh8",
+		name:    "Lsh8x8",
 		generic: true,
 	},
 	{
-		name:    "Lsh16",
+		name:    "Lsh8x16",
 		generic: true,
 	},
 	{
-		name:    "Lsh32",
+		name:    "Lsh8x32",
 		generic: true,
 	},
 	{
-		name:    "Lsh64",
+		name:    "Lsh8x64",
 		generic: true,
 	},
 	{
-		name:    "Rsh8",
+		name:    "Lsh16x8",
 		generic: true,
 	},
 	{
-		name:    "Rsh8U",
+		name:    "Lsh16x16",
 		generic: true,
 	},
 	{
-		name:    "Rsh16",
+		name:    "Lsh16x32",
 		generic: true,
 	},
 	{
-		name:    "Rsh16U",
+		name:    "Lsh16x64",
 		generic: true,
 	},
 	{
-		name:    "Rsh32",
+		name:    "Lsh32x8",
 		generic: true,
 	},
 	{
-		name:    "Rsh32U",
+		name:    "Lsh32x16",
 		generic: true,
 	},
 	{
-		name:    "Rsh64",
+		name:    "Lsh32x32",
 		generic: true,
 	},
 	{
-		name:    "Rsh64U",
+		name:    "Lsh32x64",
+		generic: true,
+	},
+	{
+		name:    "Lsh64x8",
+		generic: true,
+	},
+	{
+		name:    "Lsh64x16",
+		generic: true,
+	},
+	{
+		name:    "Lsh64x32",
+		generic: true,
+	},
+	{
+		name:    "Lsh64x64",
+		generic: true,
+	},
+	{
+		name:    "Rsh8x8",
+		generic: true,
+	},
+	{
+		name:    "Rsh8x16",
+		generic: true,
+	},
+	{
+		name:    "Rsh8x32",
+		generic: true,
+	},
+	{
+		name:    "Rsh8x64",
+		generic: true,
+	},
+	{
+		name:    "Rsh16x8",
+		generic: true,
+	},
+	{
+		name:    "Rsh16x16",
+		generic: true,
+	},
+	{
+		name:    "Rsh16x32",
+		generic: true,
+	},
+	{
+		name:    "Rsh16x64",
+		generic: true,
+	},
+	{
+		name:    "Rsh32x8",
+		generic: true,
+	},
+	{
+		name:    "Rsh32x16",
+		generic: true,
+	},
+	{
+		name:    "Rsh32x32",
+		generic: true,
+	},
+	{
+		name:    "Rsh32x64",
+		generic: true,
+	},
+	{
+		name:    "Rsh64x8",
+		generic: true,
+	},
+	{
+		name:    "Rsh64x16",
+		generic: true,
+	},
+	{
+		name:    "Rsh64x32",
+		generic: true,
+	},
+	{
+		name:    "Rsh64x64",
+		generic: true,
+	},
+	{
+		name:    "Rsh8Ux8",
+		generic: true,
+	},
+	{
+		name:    "Rsh8Ux16",
+		generic: true,
+	},
+	{
+		name:    "Rsh8Ux32",
+		generic: true,
+	},
+	{
+		name:    "Rsh8Ux64",
+		generic: true,
+	},
+	{
+		name:    "Rsh16Ux8",
+		generic: true,
+	},
+	{
+		name:    "Rsh16Ux16",
+		generic: true,
+	},
+	{
+		name:    "Rsh16Ux32",
+		generic: true,
+	},
+	{
+		name:    "Rsh16Ux64",
+		generic: true,
+	},
+	{
+		name:    "Rsh32Ux8",
+		generic: true,
+	},
+	{
+		name:    "Rsh32Ux16",
+		generic: true,
+	},
+	{
+		name:    "Rsh32Ux32",
+		generic: true,
+	},
+	{
+		name:    "Rsh32Ux64",
+		generic: true,
+	},
+	{
+		name:    "Rsh64Ux8",
+		generic: true,
+	},
+	{
+		name:    "Rsh64Ux16",
+		generic: true,
+	},
+	{
+		name:    "Rsh64Ux32",
+		generic: true,
+	},
+	{
+		name:    "Rsh64Ux64",
 		generic: true,
 	},
 	{
@@ -2315,6 +2550,22 @@ var opcodeTable = [...]opInfo{
 		name:    "Neg64",
 		generic: true,
 	},
+	{
+		name:    "Com8",
+		generic: true,
+	},
+	{
+		name:    "Com16",
+		generic: true,
+	},
+	{
+		name:    "Com32",
+		generic: true,
+	},
+	{
+		name:    "Com64",
+		generic: true,
+	},
 	{
 		name:    "Phi",
 		generic: true,

src/cmd/compile/internal/ssa/shift_test.go

@@ -10,17 +10,17 @@ import (
 
 func TestShiftConstAMD64(t *testing.T) {
 	c := NewConfig("amd64", DummyFrontend{t})
-	fun := makeConstShiftFunc(c, 18, OpLsh64, TypeUInt64)
+	fun := makeConstShiftFunc(c, 18, OpLsh64x64, TypeUInt64)
 	checkOpcodeCounts(t, fun.f, map[Op]int{OpAMD64SHLQconst: 1, OpAMD64CMPQconst: 0, OpAMD64ANDQconst: 0})
-	fun = makeConstShiftFunc(c, 66, OpLsh64, TypeUInt64)
+	fun = makeConstShiftFunc(c, 66, OpLsh64x64, TypeUInt64)
 	checkOpcodeCounts(t, fun.f, map[Op]int{OpAMD64SHLQconst: 0, OpAMD64CMPQconst: 0, OpAMD64ANDQconst: 0})
-	fun = makeConstShiftFunc(c, 18, OpRsh64U, TypeUInt64)
+	fun = makeConstShiftFunc(c, 18, OpRsh64Ux64, TypeUInt64)
 	checkOpcodeCounts(t, fun.f, map[Op]int{OpAMD64SHRQconst: 1, OpAMD64CMPQconst: 0, OpAMD64ANDQconst: 0})
-	fun = makeConstShiftFunc(c, 66, OpRsh64U, TypeUInt64)
+	fun = makeConstShiftFunc(c, 66, OpRsh64Ux64, TypeUInt64)
 	checkOpcodeCounts(t, fun.f, map[Op]int{OpAMD64SHRQconst: 0, OpAMD64CMPQconst: 0, OpAMD64ANDQconst: 0})
-	fun = makeConstShiftFunc(c, 18, OpRsh64, TypeInt64)
+	fun = makeConstShiftFunc(c, 18, OpRsh64x64, TypeInt64)
 	checkOpcodeCounts(t, fun.f, map[Op]int{OpAMD64SARQconst: 1, OpAMD64CMPQconst: 0})
-	fun = makeConstShiftFunc(c, 66, OpRsh64, TypeInt64)
+	fun = makeConstShiftFunc(c, 66, OpRsh64x64, TypeInt64)
 	checkOpcodeCounts(t, fun.f, map[Op]int{OpAMD64SARQconst: 1, OpAMD64CMPQconst: 0})
 }