1
0
mirror of https://github.com/golang/go synced 2024-11-12 05:50:21 -07:00

regexp: add one-pass optimization from RE2

This produces about a 2.3x speedup for patterns
that can be handled this way.

LGTM=rsc
R=rsc
CC=golang-codereviews
https://golang.org/cl/13345046
This commit is contained in:
David Covert 2014-03-07 15:30:02 -05:00 committed by Russ Cox
parent 84570aa9a1
commit 76236ef136
5 changed files with 952 additions and 20 deletions

View File

@ -578,3 +578,58 @@ func BenchmarkAnchoredLongMatch(b *testing.B) {
re.Match(x)
}
}
func BenchmarkOnePassShortA(b *testing.B) {
b.StopTimer()
x := []byte("abcddddddeeeededd")
re := MustCompile("^.bc(d|e)*$")
b.StartTimer()
for i := 0; i < b.N; i++ {
re.Match(x)
}
}
func BenchmarkNotOnePassShortA(b *testing.B) {
b.StopTimer()
x := []byte("abcddddddeeeededd")
re := MustCompile(".bc(d|e)*$")
b.StartTimer()
for i := 0; i < b.N; i++ {
re.Match(x)
}
}
func BenchmarkOnePassShortB(b *testing.B) {
b.StopTimer()
x := []byte("abcddddddeeeededd")
re := MustCompile("^.bc(?:d|e)*$")
b.StartTimer()
for i := 0; i < b.N; i++ {
re.Match(x)
}
}
func BenchmarkNotOnePassShortB(b *testing.B) {
b.StopTimer()
x := []byte("abcddddddeeeededd")
re := MustCompile(".bc(?:d|e)*$")
b.StartTimer()
for i := 0; i < b.N; i++ {
re.Match(x)
}
}
func BenchmarkOnePassLongPrefix(b *testing.B) {
b.StopTimer()
x := []byte("abcdefghijklmnopqrstuvwxyz")
re := MustCompile("^abcdefghijklmnopqrstuvwxyz.*$")
b.StartTimer()
for i := 0; i < b.N; i++ {
re.Match(x)
}
}
func BenchmarkOnePassLongNotPrefix(b *testing.B) {
b.StopTimer()
x := []byte("abcdefghijklmnopqrstuvwxyz")
re := MustCompile("^.bcdefghijklmnopqrstuvwxyz.*$")
b.StartTimer()
for i := 0; i < b.N; i++ {
re.Match(x)
}
}

View File

@ -37,6 +37,7 @@ type thread struct {
type machine struct {
re *Regexp // corresponding Regexp
p *syntax.Prog // compiled program
op *syntax.Prog // compiled onepass program, or syntax.NotOnePass
q0, q1 queue // two queues for runq, nextq
pool []*thread // pool of available threads
matched bool // whether a match was found
@ -66,8 +67,8 @@ func (m *machine) newInputReader(r io.RuneReader) input {
}
// progMachine returns a new machine running the prog p.
func progMachine(p *syntax.Prog) *machine {
m := &machine{p: p}
func progMachine(p, op *syntax.Prog) *machine {
m := &machine{p: p, op: op}
n := len(m.p.Inst)
m.q0 = queue{make([]uint32, n), make([]entry, 0, n)}
m.q1 = queue{make([]uint32, n), make([]entry, 0, n)}
@ -312,6 +313,105 @@ func (m *machine) add(q *queue, pc uint32, pos int, cap []int, cond syntax.Empty
return t
}
// onepass runs the machine over the input starting at pos.
// It reports whether a match was found.
// If so, m.matchcap holds the submatch information.
func (m *machine) onepass(i input, pos int) bool {
startCond := m.re.cond
if startCond == ^syntax.EmptyOp(0) { // impossible
return false
}
m.matched = false
for i := range m.matchcap {
m.matchcap[i] = -1
}
r, r1 := endOfText, endOfText
width, width1 := 0, 0
r, width = i.step(pos)
if r != endOfText {
r1, width1 = i.step(pos + width)
}
var flag syntax.EmptyOp
if pos == 0 {
flag = syntax.EmptyOpContext(-1, r)
} else {
flag = i.context(pos)
}
pc := m.op.Start
inst := m.op.Inst[pc]
// If there is a simple literal prefix, skip over it.
if pos == 0 && syntax.EmptyOp(inst.Arg)&^flag == 0 &&
len(m.re.prefix) > 0 && i.canCheckPrefix() {
// Match requires literal prefix; fast search for it.
if i.hasPrefix(m.re) {
pos += len(m.re.prefix)
r, width = i.step(pos)
r1, width1 = i.step(pos + width)
flag = i.context(pos)
pc = int(m.re.prefixEnd)
} else {
return m.matched
}
}
for {
inst = m.op.Inst[pc]
pc = int(inst.Out)
switch inst.Op {
default:
panic("bad inst")
case syntax.InstMatch:
m.matched = true
if len(m.matchcap) > 0 {
m.matchcap[0] = 0
m.matchcap[1] = pos
}
return m.matched
case syntax.InstRune:
if !inst.MatchRune(r) {
return m.matched
}
case syntax.InstRune1:
if r != inst.Rune[0] {
return m.matched
}
case syntax.InstRuneAny:
// Nothing
case syntax.InstRuneAnyNotNL:
if r == '\n' {
return m.matched
}
// peek at the input rune to see which branch of the Alt to take
case syntax.InstAlt, syntax.InstAltMatch:
pc = int(inst.OnePassNext(r))
continue
case syntax.InstFail:
return m.matched
case syntax.InstNop:
continue
case syntax.InstEmptyWidth:
if syntax.EmptyOp(inst.Arg)&^flag != 0 {
return m.matched
}
continue
case syntax.InstCapture:
if int(inst.Arg) < len(m.matchcap) {
m.matchcap[inst.Arg] = pos
}
continue
}
if width == 0 {
break
}
flag = syntax.EmptyOpContext(r, r1)
pos += width
r, width = r1, width1
if r != endOfText {
r1, width1 = i.step(pos + width)
}
}
return m.matched
}
// empty is a non-nil 0-element slice,
// so doExecute can avoid an allocation
// when 0 captures are requested from a successful match.
@ -329,16 +429,23 @@ func (re *Regexp) doExecute(r io.RuneReader, b []byte, s string, pos int, ncap i
} else {
i = m.newInputString(s)
}
m.init(ncap)
if !m.match(i, pos) {
re.put(m)
return nil
if m.op != syntax.NotOnePass {
if !m.onepass(i, pos) {
re.put(m)
return nil
}
} else {
m.init(ncap)
if !m.match(i, pos) {
re.put(m)
return nil
}
}
if ncap == 0 {
re.put(m)
return empty // empty but not nil
}
cap := make([]int, ncap)
cap := make([]int, len(m.matchcap))
copy(cap, m.matchcap)
re.put(m)
return cap

View File

@ -75,10 +75,12 @@ type Regexp struct {
// read-only after Compile
expr string // as passed to Compile
prog *syntax.Prog // compiled program
onepass *syntax.Prog // onpass program or nil
prefix string // required prefix in unanchored matches
prefixBytes []byte // prefix, as a []byte
prefixComplete bool // prefix is the entire regexp
prefixRune rune // first rune in prefix
prefixEnd uint32 // pc for last rune in prefix
cond syntax.EmptyOp // empty-width conditions required at start of match
numSubexp int
subexpNames []string
@ -155,12 +157,17 @@ func compile(expr string, mode syntax.Flags, longest bool) (*Regexp, error) {
regexp := &Regexp{
expr: expr,
prog: prog,
onepass: prog.CompileOnePass(),
numSubexp: maxCap,
subexpNames: capNames,
cond: prog.StartCond(),
longest: longest,
}
regexp.prefix, regexp.prefixComplete = prog.Prefix()
if regexp.onepass == syntax.NotOnePass {
regexp.prefix, regexp.prefixComplete = prog.Prefix()
} else {
regexp.prefix, regexp.prefixComplete, regexp.prefixEnd = prog.OnePassPrefix()
}
if regexp.prefix != "" {
// TODO(rsc): Remove this allocation by adding
// IndexString to package bytes.
@ -182,7 +189,7 @@ func (re *Regexp) get() *machine {
return z
}
re.mu.Unlock()
z := progMachine(re.prog)
z := progMachine(re.prog, re.onepass)
z.re = re
return z
}

View File

@ -6,6 +6,7 @@ package syntax
import (
"bytes"
"sort"
"strconv"
"unicode"
)
@ -35,8 +36,30 @@ const (
InstRune1
InstRuneAny
InstRuneAnyNotNL
InstLast
)
var instOpNames = []string{
"InstAlt",
"InstAltMatch",
"InstCapture",
"InstEmptyWidth",
"InstMatch",
"InstFail",
"InstNop",
"InstRune",
"InstRune1",
"InstRuneAny",
"InstRuneAnyNotNL",
}
func (i InstOp) String() string {
if i >= InstLast {
return ""
}
return instOpNames[i]
}
// An EmptyOp specifies a kind or mixture of zero-width assertions.
type EmptyOp uint8
@ -93,6 +116,7 @@ type Inst struct {
Out uint32 // all but InstMatch, InstFail
Arg uint32 // InstAlt, InstAltMatch, InstCapture, InstEmptyWidth
Rune []rune
Next []uint32 // If input rune matches
}
func (p *Prog) String() string {
@ -103,13 +127,13 @@ func (p *Prog) String() string {
// skipNop follows any no-op or capturing instructions
// and returns the resulting pc.
func (p *Prog) skipNop(pc uint32) *Inst {
func (p *Prog) skipNop(pc uint32) (*Inst, uint32) {
i := &p.Inst[pc]
for i.Op == InstNop || i.Op == InstCapture {
pc = i.Out
i = &p.Inst[pc]
}
return i
return i, pc
}
// op returns i.Op but merges all the Rune special cases into InstRune
@ -126,7 +150,7 @@ func (i *Inst) op() InstOp {
// regexp must start with. Complete is true if the prefix
// is the entire match.
func (p *Prog) Prefix() (prefix string, complete bool) {
i := p.skipNop(uint32(p.Start))
i, _ := p.skipNop(uint32(p.Start))
// Avoid allocation of buffer if prefix is empty.
if i.op() != InstRune || len(i.Rune) != 1 {
@ -137,11 +161,41 @@ func (p *Prog) Prefix() (prefix string, complete bool) {
var buf bytes.Buffer
for i.op() == InstRune && len(i.Rune) == 1 && Flags(i.Arg)&FoldCase == 0 {
buf.WriteRune(i.Rune[0])
i = p.skipNop(i.Out)
i, _ = p.skipNop(i.Out)
}
return buf.String(), i.Op == InstMatch
}
// OnePassPrefix returns a literal string that all matches for the
// regexp must start with. Complete is true if the prefix
// is the entire match. Pc is the index of the last rune instruction
// in the string. The OnePassPrefix skips over the mandatory
// EmptyBeginText
func (p *Prog) OnePassPrefix() (prefix string, complete bool, pc uint32) {
i := &p.Inst[p.Start]
if i.Op != InstEmptyWidth || (EmptyOp(i.Arg))&EmptyBeginText == 0 {
return "", i.Op == InstMatch, uint32(p.Start)
}
pc = i.Out
i = &p.Inst[pc]
for i.Op == InstNop {
pc = i.Out
i = &p.Inst[pc]
}
// Avoid allocation of buffer if prefix is empty.
if i.op() != InstRune || len(i.Rune) != 1 {
return "", i.Op == InstMatch, uint32(p.Start)
}
// Have prefix; gather characters.
var buf bytes.Buffer
for i.op() == InstRune && len(i.Rune) == 1 && Flags(i.Arg)&FoldCase == 0 {
buf.WriteRune(i.Rune[0])
pc, i = i.Out, &p.Inst[i.Out]
}
return buf.String(), i.Op == InstEmptyWidth && (EmptyOp(i.Arg))&EmptyBeginText != 0, pc
}
// StartCond returns the leading empty-width conditions that must
// be true in any match. It returns ^EmptyOp(0) if no matches are possible.
func (p *Prog) StartCond() EmptyOp {
@ -166,35 +220,58 @@ Loop:
return flag
}
const noMatch = -1
// OnePassNext selects the next actionable state of the prog, based on the input character.
// It should only be called when i.Op == InstAlt or InstAltMatch, and from the one-pass machine.
// One of the alternates may ultimately lead without input to end of line. If the instruction
// is InstAltMatch the path to the InstMatch is in i.Out, the normal node in i.Next.
func (i *Inst) OnePassNext(r rune) uint32 {
next := i.MatchRunePos(r)
if next != noMatch {
return i.Next[next]
}
if i.Op == InstAltMatch {
return i.Out
}
return 0
}
// MatchRune returns true if the instruction matches (and consumes) r.
// It should only be called when i.Op == InstRune.
func (i *Inst) MatchRune(r rune) bool {
return i.MatchRunePos(r) != noMatch
}
// MatchRunePos returns the index of the rune pair if the instruction matches.
// It should only be called when i.Op == InstRune.
func (i *Inst) MatchRunePos(r rune) int {
rune := i.Rune
// Special case: single-rune slice is from literal string, not char class.
if len(rune) == 1 {
r0 := rune[0]
if r == r0 {
return true
return 0
}
if Flags(i.Arg)&FoldCase != 0 {
for r1 := unicode.SimpleFold(r0); r1 != r0; r1 = unicode.SimpleFold(r1) {
if r == r1 {
return true
return 0
}
}
}
return false
return noMatch
}
// Peek at the first few pairs.
// Should handle ASCII well.
for j := 0; j < len(rune) && j <= 8; j += 2 {
if r < rune[j] {
return false
return noMatch
}
if r <= rune[j+1] {
return true
return j / 2
}
}
@ -205,14 +282,14 @@ func (i *Inst) MatchRune(r rune) bool {
m := lo + (hi-lo)/2
if c := rune[2*m]; c <= r {
if r <= rune[2*m+1] {
return true
return m
}
lo = m + 1
} else {
hi = m
}
}
return false
return noMatch
}
// As per re2's Prog::IsWordChar. Determines whether rune is an ASCII word char.
@ -311,3 +388,491 @@ func dumpInst(b *bytes.Buffer, i *Inst) {
bw(b, "anynotnl -> ", u32(i.Out))
}
}
// Sparse Array implementation is used as a queue.
type queue struct {
sparse []uint32
dense []uint32
size, nextIndex uint32
}
func (q *queue) empty() bool {
return q.nextIndex >= q.size
}
func (q *queue) next() (n uint32) {
n = q.dense[q.nextIndex]
q.nextIndex++
return
}
func (q *queue) clear() {
q.size = 0
q.nextIndex = 0
}
func (q *queue) reset() {
q.nextIndex = 0
}
func (q *queue) contains(u uint32) bool {
if u >= uint32(len(q.sparse)) {
return false
}
return q.sparse[u] < q.size && q.dense[q.sparse[u]] == u
}
func (q *queue) insert(u uint32) {
if !q.contains(u) {
q.insertNew(u)
}
}
func (q *queue) insertNew(u uint32) {
if u >= uint32(len(q.sparse)) {
return
}
q.sparse[u] = q.size
q.dense[q.size] = u
q.size++
}
func newQueue(size int) (q *queue) {
return &queue{
sparse: make([]uint32, size),
dense: make([]uint32, size),
}
}
// mergeRuneSets merges two non-intersecting runesets, and returns the merged result,
// and a NextIp array. The idea is that if a rune matches the OnePassRunes at index
// i, NextIp[i/2] is the target. If the input sets intersect, an empty runeset and a
// NextIp array with the single element mergeFailed is returned.
// The code assumes that both inputs contain ordered and non-intersecting rune pairs.
const mergeFailed = uint32(0xffffffff)
var (
noRune = []rune{}
noNext = []uint32{mergeFailed}
)
func mergeRuneSets(leftRunes, rightRunes *[]rune, leftPC, rightPC uint32) ([]rune, []uint32) {
leftLen := len(*leftRunes)
rightLen := len(*rightRunes)
if leftLen&0x1 != 0 || rightLen&0x1 != 0 {
panic("mergeRuneSets odd length []rune")
}
var (
lx, rx int
)
merged := make([]rune, 0)
next := make([]uint32, 0)
ok := true
defer func() {
if !ok {
merged = nil
next = nil
}
}()
ix := -1
extend := func(newLow *int, newArray *[]rune, pc uint32) bool {
if ix > 0 && (*newArray)[*newLow] <= merged[ix] {
return false
}
merged = append(merged, (*newArray)[*newLow], (*newArray)[*newLow+1])
*newLow += 2
ix += 2
next = append(next, pc)
return true
}
for lx < leftLen || rx < rightLen {
switch {
case rx >= rightLen:
ok = extend(&lx, leftRunes, leftPC)
case lx >= leftLen:
ok = extend(&rx, rightRunes, rightPC)
case (*rightRunes)[rx] < (*leftRunes)[lx]:
ok = extend(&rx, rightRunes, rightPC)
default:
ok = extend(&lx, leftRunes, leftPC)
}
if !ok {
return noRune, noNext
}
}
return merged, next
}
// cleanupOnePass drops working memory, and restores certain shortcut instructions.
func (prog *Prog) cleanupOnePass(pOriginal *Prog) {
for ix, instOriginal := range pOriginal.Inst {
switch instOriginal.Op {
case InstAlt, InstAltMatch, InstRune:
case InstCapture, InstEmptyWidth, InstNop, InstMatch, InstFail:
prog.Inst[ix].Next = nil
case InstRune1, InstRuneAny, InstRuneAnyNotNL:
prog.Inst[ix].Next = nil
prog.Inst[ix] = instOriginal
}
}
}
// onePassCopy creates a copy of the original Prog, as we'll be modifying it
func (prog *Prog) onePassCopy() *Prog {
p := &Prog{
Inst: append([]Inst{}[:], prog.Inst...),
Start: prog.Start,
NumCap: prog.NumCap,
}
for _, inst := range p.Inst {
inst.Next = make([]uint32, 0)
}
// rewrites one or more common Prog constructs that enable some otherwise
// non-onepass Progs to be onepass. A:BD (for example) means an InstAlt at
// ip A, that points to ips B & C.
// A:BC + B:DA => A:BC + B:CD
// A:BC + B:DC => A:DC + B:DC
for pc := range p.Inst {
switch p.Inst[pc].Op {
default:
continue
case InstAlt, InstAltMatch:
// A:Bx + B:Ay
p_A_Other := &p.Inst[pc].Out
p_A_Alt := &p.Inst[pc].Arg
// make sure a target is another Alt
instAlt := p.Inst[*p_A_Alt]
if !(instAlt.Op == InstAlt || instAlt.Op == InstAltMatch) {
p_A_Alt, p_A_Other = p_A_Other, p_A_Alt
instAlt = p.Inst[*p_A_Alt]
if !(instAlt.Op == InstAlt || instAlt.Op == InstAltMatch) {
continue
}
}
instOther := p.Inst[*p_A_Other]
// Analyzing both legs pointing to Alts is for another day
if instOther.Op == InstAlt || instOther.Op == InstAltMatch {
// too complicated
continue
}
// simple empty transition loop
// A:BC + B:DA => A:BC + B:DC
p_B_Alt := &p.Inst[*p_A_Alt].Out
p_B_Other := &p.Inst[*p_A_Alt].Arg
patch := false
if instAlt.Out == uint32(pc) {
patch = true
} else if instAlt.Arg == uint32(pc) {
patch = true
p_B_Alt, p_B_Other = p_B_Other, p_B_Alt
}
if patch {
*p_B_Alt = *p_A_Other
}
// empty transition to common target
// A:BC + B:DC => A:DC + B:DC
if *p_A_Other == *p_B_Alt {
*p_A_Alt = *p_B_Other
}
}
}
return p
}
// runeSlice exists to permit sorting the case-folded rune sets.
type runeSlice []rune
func (p runeSlice) Len() int { return len(p) }
func (p runeSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p runeSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
// Sort is a convenience method.
func (p runeSlice) Sort() {
sort.Sort(p)
}
// makeOnePass creates a onepass Prog, if possible. It is possible if at any alt,
// the match engine can always tell which branch to take. The routine may modify
// p if it is turned into a onepass Prog. If it isn't possible for this to be a
// onepass Prog, the Prog syntax.NotOnePass is returned. makeOnePass is resursive
// to the size of the Prog
func (p *Prog) makeOnePass() *Prog {
var (
instQueue = newQueue(len(p.Inst))
visitQueue = newQueue(len(p.Inst))
build func(uint32, *queue)
check func(uint32, map[uint32]bool) bool
onePassRunes = make([][]rune, len(p.Inst))
)
build = func(pc uint32, q *queue) {
if q.contains(pc) {
return
}
inst := p.Inst[pc]
switch inst.Op {
case InstAlt, InstAltMatch:
q.insert(inst.Out)
build(inst.Out, q)
q.insert(inst.Arg)
case InstMatch, InstFail:
default:
q.insert(inst.Out)
}
}
// check that paths from Alt instructions are unambiguous, and rebuild the new
// program as a onepass program
check = func(pc uint32, m map[uint32]bool) (ok bool) {
ok = true
inst := &p.Inst[pc]
if visitQueue.contains(pc) {
return
}
visitQueue.insert(pc)
switch inst.Op {
case InstAlt, InstAltMatch:
ok = check(inst.Out, m) && check(inst.Arg, m)
// check no-input paths to InstMatch
matchOut := m[inst.Out]
matchArg := m[inst.Arg]
if matchOut && matchArg {
ok = false
break
}
// Match on empty goes in inst.Out
if matchArg {
inst.Out, inst.Arg = inst.Arg, inst.Out
matchOut, matchArg = matchArg, matchOut
}
if matchOut {
m[pc] = true
inst.Op = InstAltMatch
}
// build a dispatch operator from the two legs of the alt.
onePassRunes[pc], inst.Next = mergeRuneSets(
&onePassRunes[inst.Out], &onePassRunes[inst.Arg], inst.Out, inst.Arg)
if len(inst.Next) > 0 && inst.Next[0] == mergeFailed {
ok = false
break
}
case InstCapture, InstNop:
ok = check(inst.Out, m)
m[pc] = m[inst.Out]
// pass matching runes back through these no-ops.
onePassRunes[pc] = append([]rune{}[:], onePassRunes[inst.Out][:]...)
inst.Next = []uint32{}
for i := len(onePassRunes[pc]) / 2; i >= 0; i-- {
inst.Next = append(inst.Next, inst.Out)
}
case InstEmptyWidth:
ok = check(inst.Out, m)
m[pc] = m[inst.Out]
onePassRunes[pc] = append([]rune{}[:], onePassRunes[inst.Out][:]...)
inst.Next = []uint32{}
for i := len(onePassRunes[pc]) / 2; i >= 0; i-- {
inst.Next = append(inst.Next, inst.Out)
}
case InstMatch, InstFail:
m[pc] = inst.Op == InstMatch
break
case InstRune:
ok = check(inst.Out, m)
m[pc] = false
if len(inst.Next) > 0 {
break
}
if len(inst.Rune) == 0 {
onePassRunes[pc] = []rune{}[:]
inst.Next = []uint32{inst.Out}
break
}
runes := make([]rune, 0)
if len(inst.Rune) == 1 && Flags(inst.Arg)&FoldCase != 0 {
r0 := inst.Rune[0]
runes = append(runes, r0, r0)
for r1 := unicode.SimpleFold(r0); r1 != r0; r1 = unicode.SimpleFold(r1) {
runes = append(runes, r1, r1)
}
sort.Sort(runeSlice(runes))
} else {
runes = append(runes, inst.Rune...)
}
onePassRunes[pc] = runes
inst.Next = []uint32{}
for i := len(onePassRunes[pc]) / 2; i >= 0; i-- {
inst.Next = append(inst.Next, inst.Out)
}
inst.Op = InstRune
case InstRune1:
ok = check(inst.Out, m)
m[pc] = false
if len(inst.Next) > 0 {
break
}
runes := []rune{}[:]
// expand case-folded runes
if Flags(inst.Arg)&FoldCase != 0 {
r0 := inst.Rune[0]
runes = append(runes, r0, r0)
for r1 := unicode.SimpleFold(r0); r1 != r0; r1 = unicode.SimpleFold(r1) {
runes = append(runes, r1, r1)
}
sort.Sort(runeSlice(runes))
} else {
runes = append(runes, inst.Rune[0], inst.Rune[0])
}
onePassRunes[pc] = runes
inst.Next = []uint32{}
for i := len(onePassRunes[pc]) / 2; i >= 0; i-- {
inst.Next = append(inst.Next, inst.Out)
}
inst.Op = InstRune
case InstRuneAny:
ok = check(inst.Out, m)
m[pc] = false
if len(inst.Next) > 0 {
break
}
onePassRunes[pc] = append([]rune{}[:], anyRune[:]...)
inst.Next = []uint32{inst.Out}
case InstRuneAnyNotNL:
ok = check(inst.Out, m)
m[pc] = false
if len(inst.Next) > 0 {
break
}
onePassRunes[pc] = append([]rune{}[:], anyRuneNotNL[:]...)
inst.Next = []uint32{}
for i := len(onePassRunes[pc]) / 2; i >= 0; i-- {
inst.Next = append(inst.Next, inst.Out)
}
}
return
}
instQueue.clear()
instQueue.insert(uint32(p.Start))
m := make(map[uint32]bool, len(p.Inst))
for !instQueue.empty() {
pc := instQueue.next()
inst := p.Inst[pc]
visitQueue.clear()
if !check(uint32(pc), m) {
p = NotOnePass
break
}
switch inst.Op {
case InstAlt, InstAltMatch:
instQueue.insert(inst.Out)
instQueue.insert(inst.Arg)
case InstCapture, InstEmptyWidth, InstNop:
instQueue.insert(inst.Out)
case InstMatch:
case InstFail:
case InstRune, InstRune1, InstRuneAny, InstRuneAnyNotNL:
default:
}
}
if p != NotOnePass {
for i, _ := range p.Inst {
p.Inst[i].Rune = onePassRunes[i][:]
}
}
return p
}
// walk visits each Inst in the prog once, and applies the argument
// function(ip, next), in pre-order.
func (prog *Prog) walk(funcs ...func(ip, next uint32)) {
var walk1 func(uint32)
progQueue := newQueue(len(prog.Inst))
walk1 = func(ip uint32) {
if progQueue.contains(ip) {
return
}
progQueue.insert(ip)
inst := prog.Inst[ip]
switch inst.Op {
case InstAlt, InstAltMatch:
for _, f := range funcs {
f(ip, inst.Out)
f(ip, inst.Arg)
}
walk1(inst.Out)
walk1(inst.Arg)
default:
for _, f := range funcs {
f(ip, inst.Out)
}
walk1(inst.Out)
}
}
walk1(uint32(prog.Start))
}
// find returns the Insts that match the argument predicate function
func (prog *Prog) find(f func(*Prog, int) bool) (matches []uint32) {
matches = []uint32{}
for ip := range prog.Inst {
if f(prog, ip) {
matches = append(matches, uint32(ip))
}
}
return
}
var NotOnePass *Prog = nil
var debug = false
// CompileOnePass returns a new *Prog suitable for onePass execution if the original Prog
// can be recharacterized as a one-pass regexp program, or syntax.NotOnePass if the
// Prog cannot be converted. For a one pass prog, the fundamental condition that must
// be true is: at any InstAlt, there must be no ambiguity about what branch to take.
func (prog *Prog) CompileOnePass() (p *Prog) {
if prog.Start == 0 {
return NotOnePass
}
// onepass regexp is anchored
if prog.Inst[prog.Start].Op != InstEmptyWidth ||
EmptyOp(prog.Inst[prog.Start].Arg)&EmptyBeginText != EmptyBeginText {
return NotOnePass
}
// every instruction leading to InstMatch must be EmptyEndText
for _, inst := range prog.Inst {
opOut := prog.Inst[inst.Out].Op
switch inst.Op {
default:
if opOut == InstMatch {
return NotOnePass
}
case InstAlt, InstAltMatch:
if opOut == InstMatch || prog.Inst[inst.Arg].Op == InstMatch {
return NotOnePass
}
case InstEmptyWidth:
if opOut == InstMatch {
if EmptyOp(inst.Arg)&EmptyEndText == EmptyEndText {
continue
}
return NotOnePass
}
}
}
// Creates a slightly optimized copy of the original Prog
// that cleans up some Prog idioms that block valid onepass programs
p = prog.onePassCopy()
// checkAmbiguity on InstAlts, build onepass Prog if possible
p = p.makeOnePass()
if p != NotOnePass {
p.cleanupOnePass(prog)
}
return p
}

View File

@ -5,6 +5,7 @@
package syntax
import (
"reflect"
"testing"
)
@ -114,3 +115,200 @@ func BenchmarkEmptyOpContext(b *testing.B) {
EmptyOpContext(r1, -1)
}
}
var runeMergeTests = []struct {
left, right, merged []rune
next []uint32
leftPC, rightPC uint32
}{
{
// empty rhs
[]rune{69, 69},
[]rune{},
[]rune{69, 69},
[]uint32{1},
1, 2,
},
{
// identical runes, identical targets
[]rune{69, 69},
[]rune{69, 69},
[]rune{},
[]uint32{mergeFailed},
1, 1,
},
{
// identical runes, different targets
[]rune{69, 69},
[]rune{69, 69},
[]rune{},
[]uint32{mergeFailed},
1, 2,
},
{
// append right-first
[]rune{69, 69},
[]rune{71, 71},
[]rune{69, 69, 71, 71},
[]uint32{1, 2},
1, 2,
},
{
// append, left-first
[]rune{71, 71},
[]rune{69, 69},
[]rune{69, 69, 71, 71},
[]uint32{2, 1},
1, 2,
},
{
// successful interleave
[]rune{60, 60, 71, 71, 101, 101},
[]rune{69, 69, 88, 88},
[]rune{60, 60, 69, 69, 71, 71, 88, 88, 101, 101},
[]uint32{1, 2, 1, 2, 1},
1, 2,
},
{
// left surrounds right
[]rune{69, 74},
[]rune{71, 71},
[]rune{},
[]uint32{mergeFailed},
1, 2,
},
{
// right surrounds left
[]rune{69, 74},
[]rune{68, 75},
[]rune{},
[]uint32{mergeFailed},
1, 2,
},
{
// overlap at interval begin
[]rune{69, 74},
[]rune{74, 75},
[]rune{},
[]uint32{mergeFailed},
1, 2,
},
{
// overlap ar interval end
[]rune{69, 74},
[]rune{65, 69},
[]rune{},
[]uint32{mergeFailed},
1, 2,
},
{
// overlap from above
[]rune{69, 74},
[]rune{71, 74},
[]rune{},
[]uint32{mergeFailed},
1, 2,
},
{
// overlap from below
[]rune{69, 74},
[]rune{65, 71},
[]rune{},
[]uint32{mergeFailed},
1, 2,
},
{
// out of order []rune
[]rune{69, 74, 60, 65},
[]rune{66, 67},
[]rune{},
[]uint32{mergeFailed},
1, 2,
},
}
func TestMergeRuneSet(t *testing.T) {
for ix, test := range runeMergeTests {
merged, next := mergeRuneSets(&test.left, &test.right, test.leftPC, test.rightPC)
if !reflect.DeepEqual(merged, test.merged) {
t.Errorf("mergeRuneSet :%d (%v, %v) merged\n have\n%v\nwant\n%v", ix, test.left, test.right, merged, test.merged)
}
if !reflect.DeepEqual(next, test.next) {
t.Errorf("mergeRuneSet :%d(%v, %v) next\n have\n%v\nwant\n%v", ix, test.left, test.right, next, test.next)
}
}
}
const noStr = `!`
var onePass = &Prog{}
var onePassTests = []struct {
re string
onePass *Prog
prog string
}{
{`^(?:a|(?:a*))$`, NotOnePass, noStr},
{`^(?:(a)|(?:a*))$`, NotOnePass, noStr},
{`^(?:(?:(?:.(?:$))?))$`, onePass, `a`},
{`^abcd$`, onePass, `abcd`},
{`^abcd$`, onePass, `abcde`},
{`^(?:(?:a{0,})*?)$`, onePass, `a`},
{`^(?:(?:a+)*)$`, onePass, ``},
{`^(?:(?:a|(?:aa)))$`, onePass, ``},
{`^(?:[^\s\S])$`, onePass, ``},
{`^(?:(?:a{3,4}){0,})$`, NotOnePass, `aaaaaa`},
{`^(?:(?:a+)*)$`, onePass, `a`},
{`^(?:(?:(?:a*)+))$`, onePass, noStr},
{`^(?:(?:a+)*)$`, onePass, ``},
{`^[a-c]+$`, onePass, `abc`},
{`^[a-c]*$`, onePass, `abcdabc`},
{`^(?:a*)$`, onePass, `aaaaaaa`},
{`^(?:(?:aa)|a)$`, onePass, `a`},
{`^[a-c]*`, NotOnePass, `abcdabc`},
{`^[a-c]*$`, onePass, `abc`},
{`^...$`, onePass, ``},
{`^(?:a|(?:aa))$`, onePass, `a`},
{`^[a-c]*`, NotOnePass, `abcabc`},
{`^a((b))c$`, onePass, noStr},
{`^a.[l-nA-Cg-j]?e$`, onePass, noStr},
{`^a((b))$`, onePass, noStr},
{`^a(?:(b)|(c))c$`, onePass, noStr},
{`^a(?:(b*)|(c))c$`, NotOnePass, noStr},
{`^a(?:b|c)$`, onePass, noStr},
{`^a(?:b?|c)$`, onePass, noStr},
{`^a(?:b?|c?)$`, NotOnePass, noStr},
{`^a(?:b?|c+)$`, onePass, noStr},
{`^a(?:b+|(bc))d$`, NotOnePass, noStr},
{`^a(?:bc)+$`, onePass, noStr},
{`^a(?:[bcd])+$`, onePass, noStr},
{`^a((?:[bcd])+)$`, onePass, noStr},
{`^a(:?b|c)*d$`, onePass, `abbbccbbcbbd"`},
{`^.bc(d|e)*$`, onePass, `abcddddddeeeededd`},
{`^(?:(?:aa)|.)$`, NotOnePass, `a`},
{`^(?:(?:a{1,2}){1,2})$`, NotOnePass, `aaaa`},
}
func TestCompileOnePass(t *testing.T) {
var (
p *Prog
re *Regexp
err error
)
for _, test := range onePassTests {
if re, err = Parse(test.re, Perl); err != nil {
t.Errorf("Parse(%q) got err:%s, want success", test.re, err)
continue
}
// needs to be done before compile...
re = re.Simplify()
if p, err = Compile(re); err != nil {
t.Errorf("Compile(%q) got err:%s, want success", test.re, err)
continue
}
onePass = p.CompileOnePass()
if (onePass == NotOnePass) != (test.onePass == NotOnePass) {
t.Errorf("CompileOnePass(%q) got %v, expected %v", test.re, onePass, test.onePass)
}
}
}