qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-19 17:14:44 -07:00
Code Releases Activity

cmd/compile/internal/ssa: inline memmove with known size

Browse Source 
Replace calls to memmove with known (constant) size, with OpMove.
Do it only if it is safe from aliasing point of view.
Helps with code like this:

append(buf,"const str"...)

In strconv this provides nice benefit:
Quote-6                                   731ns ± 2%   647ns ± 3%  -11.41%  (p=0.000 n=10+10)
QuoteRune-6                               117ns ± 5%   111ns ± 1%   -4.54%  (p=0.000 n=10+10)
AppendQuote-6                             475ns ± 0%   396ns ± 0%  -16.59%  (p=0.000 n=9+10)
AppendQuoteRune-6                        32.0ns ± 0%  27.4ns ± 0%  -14.41%  (p=0.000 n=8+9)

Change-Id: I7704f5c51b46aed2d8f033de74c75140fc35036c
Reviewed-on: https://go-review.googlesource.com/54394
Run-TryBot: Ilya Tocar <ilya.tocar@intel.com>
Reviewed-by: Keith Randall <khr@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
This commit is contained in:
Ilya Tocar 2017-08-09 14:00:38 -05:00
parent ad3742f4ac
commit f3884680fc
6 changed files with 189 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
src/cmd/compile/internal/gc/asm_test.go
View File

@ -1191,6 +1191,36 @@ var linuxAMD64Tests = []*asmTest{
`,
pos: []string{"\tANDQ\t\\$4095,"},
},
{
// Test that small memmove was replaced with direct movs
fn: `
func $() {
x := [...]byte{1, 2, 3, 4, 5, 6, 7}
copy(x[1:], x[:])
}
`,
neg: []string{"memmove"},
},
{
// Same as above but with different size
fn: `
func $() {
x := [...]byte{1, 2, 3, 4}
copy(x[1:], x[:])
}
`,
neg: []string{"memmove"},
},
{
// Same as above but with different size
fn: `
func $() {
x := [...]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
copy(x[1:], x[:])
}
`,
neg: []string{"memmove"},
},
}
var linux386Tests = []*asmTest{
@ -1322,6 +1352,26 @@ var linux386Tests = []*asmTest{
`,
pos: []string{"\tANDL\t\\$4095,"},
},
{
// Test that small memmove was replaced with direct movs
fn: `
func $() {
x := [...]byte{1, 2, 3, 4, 5, 6, 7}
copy(x[1:], x[:])
}
`,
neg: []string{"memmove"},
},
{
// Same as above but with different size
fn: `
func $() {
x := [...]byte{1, 2, 3, 4}
copy(x[1:], x[:])
}
`,
neg: []string{"memmove"},
},
}
var linuxS390XTests = []*asmTest{

8
src/cmd/compile/internal/ssa/gen/generic.rules
View File

@ -1443,6 +1443,14 @@
(EqPtr x x) -> (ConstBool [1])
(EqPtr (Addr {a} x) (Addr {b} x)) -> (ConstBool [b2i(a == b)])
// Inline small runtime.memmove calls with constant length.
(StaticCall {sym} s1:(Store _ (Const64 [sz]) s2:(Store _ src s3:(Store {t} _ dst mem))))
&& isSameSym(sym,"runtime.memmove") && s1.Uses == 1 && s2.Uses == 1 && s3.Uses == 1 && isInlinableMemmoveSize(sz,config)
-> (Move {t.(*types.Type).Elem()} [sz] dst src mem)
(StaticCall {sym} s1:(Store _ (Const32 [sz]) s2:(Store _ src s3:(Store {t} _ dst mem))))
&& isSameSym(sym,"runtime.memmove") && s1.Uses == 1 && s2.Uses == 1 && s3.Uses == 1 && isInlinableMemmoveSize(sz,config)
-> (Move {t.(*types.Type).Elem()} [sz] dst src mem)
// De-virtualize interface calls into static calls.
// Note that (ITab (IMake)) doesn't get
// rewritten until after the first opt pass,

7
src/cmd/compile/internal/ssa/gen/genericOps.go
View File

@ -321,8 +321,11 @@ var genericOps = []opData{
{name: "Invalid"}, // unused value
// Memory operations
{name: "Load", argLength: 2}, // Load from arg0. arg1=memory
{name: "Store", argLength: 3, typ: "Mem", aux: "Typ"}, // Store arg1 to arg0. arg2=memory, aux=type. Returns memory.
{name: "Load", argLength: 2}, // Load from arg0. arg1=memory
{name: "Store", argLength: 3, typ: "Mem", aux: "Typ"}, // Store arg1 to arg0. arg2=memory, aux=type. Returns memory.
// The source and destination of Move may overlap in some cases. See e.g.
// memmove inlining in generic.rules. When inlineablememmovesize (in ../rewrite.go)
// returns true, we must do all loads before all stores, when lowering Move.
{name: "Move", argLength: 3, typ: "Mem", aux: "TypSize"}, // arg0=destptr, arg1=srcptr, arg2=mem, auxint=size, aux=type. Returns memory.
{name: "Zero", argLength: 2, typ: "Mem", aux: "TypSize"}, // arg0=destptr, arg1=mem, auxint=size, aux=type. Returns memory.

15
src/cmd/compile/internal/ssa/rewrite.go
View File

@ -668,3 +668,18 @@ func zeroUpper32Bits(x *Value, depth int) bool {
}
return false
}
// inlineablememmovesize reports whether the given arch performs OpMove of the given size
// faster than memmove and in a safe way when src and dst overlap.
// This is used as a check for replacing memmove with OpMove.
func isInlinableMemmoveSize(sz int64, c *Config) bool {
switch c.arch {
case "amd64", "amd64p32":
return sz <= 16
case "386", "ppc64", "s390x", "ppc64le":
return sz <= 8
case "arm", "mips", "mips64", "mipsle", "mips64le":
return sz <= 4
}
return false
}

22
src/cmd/compile/internal/ssa/rewrite_test.go
View File

@ -103,3 +103,25 @@ func TestLog2(t *testing.T) {
}
}
}
// We generate memmove for copy(x[1:], x[:]), however we may change it to OpMove,
// because size is known. Check that OpMove is alias-safe, or we did call memmove.
func TestMove(t *testing.T) {
x := [...]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40}
copy(x[1:], x[:])
for i := 1; i < len(x); i++ {
if int(x[i]) != i {
t.Errorf("Memmove got converted to OpMove in alias-unsafe way. Got %d insted of %d in position %d", int(x[i]), i, i+1)
}
}
}
func TestMoveSmall(t *testing.T) {
x := [...]byte{1, 2, 3, 4, 5, 6, 7}
copy(x[1:], x[:])
for i := 1; i < len(x); i++ {
if int(x[i]) != i {
t.Errorf("Memmove got converted to OpMove in alias-unsafe way. Got %d instead of %d in position %d", int(x[i]), i, i+1)
}
}
}

89
src/cmd/compile/internal/ssa/rewritegeneric.go
View File

@ -409,6 +409,8 @@ func rewriteValuegeneric(v *Value) bool {
return rewriteValuegeneric_OpSlicemask_0(v)
case OpSqrt:
return rewriteValuegeneric_OpSqrt_0(v)
case OpStaticCall:
return rewriteValuegeneric_OpStaticCall_0(v)
case OpStore:
return rewriteValuegeneric_OpStore_0(v) || rewriteValuegeneric_OpStore_10(v)
case OpStringLen:
@ -23732,6 +23734,93 @@ func rewriteValuegeneric_OpSqrt_0(v *Value) bool {
}
return false
}
func rewriteValuegeneric_OpStaticCall_0(v *Value) bool {
b := v.Block
_ = b
config := b.Func.Config
_ = config
// match: (StaticCall {sym} s1:(Store _ (Const64 [sz]) s2:(Store _ src s3:(Store {t} _ dst mem))))
// cond: isSameSym(sym,"runtime.memmove") && s1.Uses == 1 && s2.Uses == 1 && s3.Uses == 1 && isInlinableMemmoveSize(sz,config)
// result: (Move {t.(*types.Type).Elem()} [sz] dst src mem)
for {
sym := v.Aux
s1 := v.Args[0]
if s1.Op != OpStore {
break
}
_ = s1.Args[2]
s1_1 := s1.Args[1]
if s1_1.Op != OpConst64 {
break
}
sz := s1_1.AuxInt
s2 := s1.Args[2]
if s2.Op != OpStore {
break
}
_ = s2.Args[2]
src := s2.Args[1]
s3 := s2.Args[2]
if s3.Op != OpStore {
break
}
t := s3.Aux
_ = s3.Args[2]
dst := s3.Args[1]
mem := s3.Args[2]
if !(isSameSym(sym, "runtime.memmove") && s1.Uses == 1 && s2.Uses == 1 && s3.Uses == 1 && isInlinableMemmoveSize(sz, config)) {
break
}
v.reset(OpMove)
v.AuxInt = sz
v.Aux = t.(*types.Type).Elem()
v.AddArg(dst)
v.AddArg(src)
v.AddArg(mem)
return true
}
// match: (StaticCall {sym} s1:(Store _ (Const32 [sz]) s2:(Store _ src s3:(Store {t} _ dst mem))))
// cond: isSameSym(sym,"runtime.memmove") && s1.Uses == 1 && s2.Uses == 1 && s3.Uses == 1 && isInlinableMemmoveSize(sz,config)
// result: (Move {t.(*types.Type).Elem()} [sz] dst src mem)
for {
sym := v.Aux
s1 := v.Args[0]
if s1.Op != OpStore {
break
}
_ = s1.Args[2]
s1_1 := s1.Args[1]
if s1_1.Op != OpConst32 {
break
}
sz := s1_1.AuxInt
s2 := s1.Args[2]
if s2.Op != OpStore {
break
}
_ = s2.Args[2]
src := s2.Args[1]
s3 := s2.Args[2]
if s3.Op != OpStore {
break
}
t := s3.Aux
_ = s3.Args[2]
dst := s3.Args[1]
mem := s3.Args[2]
if !(isSameSym(sym, "runtime.memmove") && s1.Uses == 1 && s2.Uses == 1 && s3.Uses == 1 && isInlinableMemmoveSize(sz, config)) {
break
}
v.reset(OpMove)
v.AuxInt = sz
v.Aux = t.(*types.Type).Elem()
v.AddArg(dst)
v.AddArg(src)
v.AddArg(mem)
return true
}
return false
}
func rewriteValuegeneric_OpStore_0(v *Value) bool {
b := v.Block
_ = b