1
0
mirror of https://github.com/golang/go synced 2024-11-18 08:54:45 -07:00

internal/lsp: trim down the fuzzy matcher library

Remove the input type option. Now everything behaves as "symbol".

We don't use the "text" or "filename" input types, and I don't foresee
us using them. Removing them simplifies the code a bit, but simplifies
the tests a lot. It was tedious to make changes to the matcher logic
because you had to fret over test failure details that didn't actually
matter because we didn't use that functionality.

Change-Id: I651debde9e63ee283d7bc3ad718d22f4b9a127c0
Reviewed-on: https://go-review.googlesource.com/c/tools/+/202637
Reviewed-by: Rebecca Stambler <rstambler@golang.org>
Run-TryBot: Rebecca Stambler <rstambler@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
This commit is contained in:
Muir Manders 2019-10-21 21:07:21 -07:00 committed by Rebecca Stambler
parent 25e800de08
commit a12cc76b85
6 changed files with 74 additions and 218 deletions

View File

@ -8,19 +8,6 @@ import (
"unicode"
)
// Input specifies the type of the input. This influences how the runes are interpreted wrt to
// segmenting the input.
type Input int
const (
// Text represents a text input type. Input is not segmented.
Text Input = iota
// Filename represents a filepath input type with '/' segment delimitors.
Filename
// Symbol represents a symbol input type with '.' and ':' segment delimitors.
Symbol
)
// RuneRole specifies the role of a rune in the context of an input.
type RuneRole byte
@ -40,7 +27,7 @@ const (
// RuneRoles detects the roles of each byte rune in an input string and stores it in the output
// slice. The rune role depends on the input type. Stops when it parsed all the runes in the string
// or when it filled the output. If output is nil, then it gets created.
func RuneRoles(str string, input Input, reuse []RuneRole) []RuneRole {
func RuneRoles(str string, reuse []RuneRole) []RuneRole {
var output []RuneRole
if cap(reuse) < len(str) {
output = make([]RuneRole, 0, len(str))
@ -78,12 +65,8 @@ func RuneRoles(str string, input Input, reuse []RuneRole) []RuneRole {
}
}
} else if curr == rtPunct {
switch {
case input == Filename && r == '/':
role = RSep
case input == Symbol && r == '.':
role = RSep
case input == Symbol && r == ':':
switch r {
case '.', ':':
role = RSep
}
}

View File

@ -13,38 +13,16 @@ import (
)
var rolesTests = []struct {
str string
input fuzzy.Input
want string
str string
want string
}{
{str: "abc", want: "Ccc", input: fuzzy.Text},
{str: ".abc", want: " Ccc", input: fuzzy.Text},
{str: "abc def", want: "Ccc Ccc", input: fuzzy.Text},
{str: "SWT MyID", want: "Cuu CcCu", input: fuzzy.Text},
{str: "ID", want: "Cu", input: fuzzy.Text},
{str: "IDD", want: "Cuu", input: fuzzy.Text},
{str: " ID ", want: " Cu ", input: fuzzy.Text},
{str: "IDSome", want: "CuCccc", input: fuzzy.Text},
{str: "0123456789", want: "Cccccccccc", input: fuzzy.Text},
{str: "abcdefghigklmnopqrstuvwxyz", want: "Cccccccccccccccccccccccccc", input: fuzzy.Text},
{str: "ABCDEFGHIGKLMNOPQRSTUVWXYZ", want: "Cuuuuuuuuuuuuuuuuuuuuuuuuu", input: fuzzy.Text},
{str: "こんにちは", want: "Ccccccccccccccc", input: fuzzy.Text}, // We don't parse unicode
{str: ":/.", want: " ", input: fuzzy.Text},
// Filenames
{str: "abc/def", want: "Ccc/Ccc", input: fuzzy.Filename},
{str: " abc_def", want: " Ccc Ccc", input: fuzzy.Filename},
{str: " abc_DDf", want: " Ccc CCc", input: fuzzy.Filename},
{str: ":.", want: " ", input: fuzzy.Filename},
// Symbols
{str: "abc::def::goo", want: "Ccc//Ccc//Ccc", input: fuzzy.Symbol},
{str: "proto::Message", want: "Ccccc//Ccccccc", input: fuzzy.Symbol},
{str: "AbstractSWTFactory", want: "CcccccccCuuCcccccc", input: fuzzy.Symbol},
{str: "Abs012", want: "Cccccc", input: fuzzy.Symbol},
{str: "/", want: " ", input: fuzzy.Symbol},
{str: "fOO", want: "CCu", input: fuzzy.Symbol},
{str: "fo_oo.o_oo", want: "Cc Cc/C Cc", input: fuzzy.Symbol},
{str: "abc::def::goo", want: "Ccc//Ccc//Ccc"},
{str: "proto::Message", want: "Ccccc//Ccccccc"},
{str: "AbstractSWTFactory", want: "CcccccccCuuCcccccc"},
{str: "Abs012", want: "Cccccc"},
{str: "/", want: " "},
{str: "fOO", want: "CCu"},
{str: "fo_oo.o_oo", want: "Cc Cc/C Cc"},
}
func rolesString(roles []fuzzy.RuneRole) string {
@ -58,7 +36,7 @@ func rolesString(roles []fuzzy.RuneRole) string {
func TestRoles(t *testing.T) {
for _, tc := range rolesTests {
gotRoles := make([]fuzzy.RuneRole, len(tc.str))
fuzzy.RuneRoles(tc.str, tc.input, gotRoles)
fuzzy.RuneRoles(tc.str, gotRoles)
got := rolesString(gotRoles)
if got != tc.want {
t.Errorf("roles(%s) = %v; want %v", tc.str, got, tc.want)
@ -98,7 +76,7 @@ var wordSplitTests = []struct {
func TestWordSplit(t *testing.T) {
for _, tc := range wordSplitTests {
roles := fuzzy.RuneRoles(tc.input, fuzzy.Symbol, nil)
roles := fuzzy.RuneRoles(tc.input, nil)
var got []string
consumer := func(i, j int) {
@ -127,45 +105,30 @@ func diffStringLists(a, b []string) bool {
}
var lastSegmentSplitTests = []struct {
str string
input fuzzy.Input
want string
str string
want string
}{
{
str: "identifier",
input: fuzzy.Symbol,
want: "identifier",
str: "identifier",
want: "identifier",
},
{
str: "two_words",
input: fuzzy.Symbol,
want: "two_words",
str: "two_words",
want: "two_words",
},
{
str: "first::second",
input: fuzzy.Symbol,
want: "second",
str: "first::second",
want: "second",
},
{
str: "foo.bar.FOOBar_buz123_test",
input: fuzzy.Symbol,
want: "FOOBar_buz123_test",
},
{
str: "golang.org/x/tools/internal/lsp/fuzzy_matcher.go",
input: fuzzy.Filename,
want: "fuzzy_matcher.go",
},
{
str: "golang.org/x/tools/internal/lsp/fuzzy_matcher.go",
input: fuzzy.Text,
want: "golang.org/x/tools/internal/lsp/fuzzy_matcher.go",
str: "foo.bar.FOOBar_buz123_test",
want: "FOOBar_buz123_test",
},
}
func TestLastSegment(t *testing.T) {
for _, tc := range lastSegmentSplitTests {
roles := fuzzy.RuneRoles(tc.str, tc.input, nil)
roles := fuzzy.RuneRoles(tc.str, nil)
got := fuzzy.LastSegment(tc.str, roles)
@ -180,7 +143,7 @@ func BenchmarkRoles(b *testing.B) {
out := make([]fuzzy.RuneRole, len(str))
for i := 0; i < b.N; i++ {
fuzzy.RuneRoles(str, fuzzy.Symbol, out)
fuzzy.RuneRoles(str, out)
}
b.SetBytes(int64(len(str)))
}

View File

@ -36,8 +36,6 @@ func score(val int, prevK int /*0 or 1*/) scoreVal {
// Matcher implements a fuzzy matching algorithm for scoring candidates against a pattern.
// The matcher does not support parallel usage.
type Matcher struct {
input Input
pattern string
patternLower []byte // lower-case version of the pattern
patternShort []byte // first characters of the pattern
@ -67,13 +65,12 @@ func (m *Matcher) bestK(i, j int) int {
}
// NewMatcher returns a new fuzzy matcher for scoring candidates against the provided pattern.
func NewMatcher(pattern string, input Input) *Matcher {
func NewMatcher(pattern string) *Matcher {
if len(pattern) > MaxPatternSize {
pattern = pattern[:MaxPatternSize]
}
m := &Matcher{
input: input,
pattern: pattern,
patternLower: ToLower(pattern, nil),
}
@ -91,28 +88,16 @@ func NewMatcher(pattern string, input Input) *Matcher {
m.patternShort = m.patternLower
}
m.patternRoles = RuneRoles(pattern, input, nil)
m.patternRoles = RuneRoles(pattern, nil)
if len(pattern) > 0 {
maxCharScore := 4
if input == Text {
maxCharScore = 6
}
m.scoreScale = 1 / float32(maxCharScore*len(pattern))
}
return m
}
// SetInput updates the input type for subsequent scoring attempts.
func (m *Matcher) SetInput(input Input) {
if m.input == input {
return
}
m.input = input
m.patternRoles = RuneRoles(m.pattern, input, m.patternRoles)
}
// Score returns the score returned by matching the candidate to the pattern.
// This is not designed for parallel use. Multiple candidates must be scored sequentially.
// Returns a score between 0 and 1 (0 - no match, 1 - perfect match).
@ -202,7 +187,7 @@ func (m *Matcher) match(candidate string, candidateLower []byte) bool {
// The input passes the simple test against pattern, so it is time to classify its characters.
// Character roles are used below to find the last segment.
m.roles = RuneRoles(candidate, m.input, m.rolesBuf[:])
m.roles = RuneRoles(candidate, m.rolesBuf[:])
return true
}
@ -226,10 +211,6 @@ func (m *Matcher) computeScore(candidate string, candidateLower []byte) int {
// A per-character bonus for a consecutive match.
consecutiveBonus := 2
if m.input == Text {
// Consecutive matches for text are more important.
consecutiveBonus = 4
}
wordIdx := 0 // Word count within segment.
for i := 1; i <= candLen; i++ {
@ -244,7 +225,7 @@ func (m *Matcher) computeScore(candidate string, candidateLower []byte) int {
}
var skipPenalty int
if segmentsLeft == 1 && isHead && m.input != Text {
if segmentsLeft == 1 && isHead {
// Skipping a word.
skipPenalty++
}
@ -317,7 +298,7 @@ func (m *Matcher) computeScore(candidate string, candidateLower []byte) int {
sc := m.scores[i-1][j-1][k].val() + charScore
isConsecutive := k == 1 || i-1 == 0 || i-1 == lastSegStart
if isConsecutive || (m.input == Text && j-1 == 0) {
if isConsecutive {
// Bonus 3: a consecutive match. First character match also gets a bonus to
// ensure prefix final match score normalizes to 1.0.
// Logically, this is a part of charScore, but we have to compute it here because it

View File

@ -17,26 +17,6 @@ import (
"golang.org/x/tools/internal/lsp/fuzzy"
)
func ExampleMatcher() {
pattern := "TEdit"
candidates := []string{"fuzzy.TextEdit", "ArtEdit", "TED talks about IT"}
// Create a fuzzy matcher for the pattern.
matcher := fuzzy.NewMatcher(pattern, fuzzy.Text)
for _, candidate := range candidates {
// Compute candidate's score against the matcher.
score := matcher.Score(candidate)
if score > -1 {
// Get the substrings in the candidate matching the pattern.
ranges := matcher.MatchedRanges()
fmt.Println(ranges) // Do something with the ranges.
}
}
}
type comparator struct {
f func(val, ref float32) bool
descr string
@ -73,12 +53,10 @@ type scoreTest struct {
var matcherTests = []struct {
pattern string
input fuzzy.Input
tests []scoreTest
}{
{
pattern: "",
input: fuzzy.Text,
tests: []scoreTest{
{"def", eq, 1},
{"Ab stuff c", eq, 1},
@ -86,7 +64,6 @@ var matcherTests = []struct {
},
{
pattern: "abc",
input: fuzzy.Text,
tests: []scoreTest{
{"def", eq, -1},
{"abd", eq, -1},
@ -97,7 +74,6 @@ var matcherTests = []struct {
},
{
pattern: "Abc",
input: fuzzy.Text,
tests: []scoreTest{
{"def", eq, -1},
{"abd", eq, -1},
@ -106,29 +82,11 @@ var matcherTests = []struct {
{"Ab stuff c", ge, 0},
},
},
{
pattern: "subs",
input: fuzzy.Filename,
tests: []scoreTest{
{"sub/seq", ge, 0},
{"sub/seq/end", ge, 0},
{"sub/seq/base", ge, 0},
},
},
{
pattern: "subs",
input: fuzzy.Filename,
tests: []scoreTest{
{"//sub/seq", ge, 0},
{"//sub/seq/end", ge, 0},
{"//sub/seq/base", ge, 0},
},
},
}
func TestScore(t *testing.T) {
for _, tc := range matcherTests {
m := fuzzy.NewMatcher(tc.pattern, tc.input)
m := fuzzy.NewMatcher(tc.pattern)
for _, sct := range tc.tests {
score := m.Score(sct.candidate)
if !sct.comparator.eval(score, sct.ref) {
@ -146,34 +104,18 @@ type candidateCompTest struct {
var compareCandidatesTestCases = []struct {
pattern string
input fuzzy.Input
orderedCandidates []string
}{
{
pattern: "aa",
input: fuzzy.Filename,
orderedCandidates: []string{
"baab",
"bb_aa",
"a/a/a",
"aa_bb",
"aa_b",
"aabb",
"aab",
"b/aa",
},
},
{
pattern: "Foo",
input: fuzzy.Text,
orderedCandidates: []string{
"Barfoo",
"F_o_o",
"Faoo",
"F__oo",
"F_oo",
"FaoFooa",
"BarFoo",
"Faoo",
"F_o_o",
"F__oo",
"F_oo",
"FooA",
"FooBar",
"Foo",
@ -183,7 +125,7 @@ var compareCandidatesTestCases = []struct {
func TestCompareCandidateScores(t *testing.T) {
for _, tc := range compareCandidatesTestCases {
m := fuzzy.NewMatcher(tc.pattern, tc.input)
m := fuzzy.NewMatcher(tc.pattern)
var prevScore float32
prevCand := "MIN_SCORE"
@ -202,55 +144,42 @@ func TestCompareCandidateScores(t *testing.T) {
}
var fuzzyMatcherTestCases = []struct {
p string
str string
want string
input fuzzy.Input
p string
str string
want string
}{
// fuzzy.Filename
{p: "aa", str: "a_a/a_a", want: "[a]_a/[a]_a", input: fuzzy.Filename},
{p: "aaaa", str: "a_a/a_a", want: "[a]_[a]/[a]_[a]", input: fuzzy.Filename},
{p: "aaaa", str: "aaaa", want: "[aaaa]", input: fuzzy.Filename},
{p: "aaaa", str: "a_a/a_aaaa", want: "a_a/[a]_[aaa]a", input: fuzzy.Filename},
{p: "aaaa", str: "a_a/aaaaa", want: "a_a/[aaaa]a", input: fuzzy.Filename},
{p: "aaaa", str: "aabaaa", want: "[aa]b[aa]a", input: fuzzy.Filename},
{p: "aaaa", str: "a/baaa", want: "[a]/b[aaa]", input: fuzzy.Filename},
{p: "abcxz", str: "d/abc/abcd/oxz", want: "d/[abc]/abcd/o[xz]", input: fuzzy.Filename},
{p: "abcxz", str: "d/abcd/abc/oxz", want: "d/[abc]d/abc/o[xz]", input: fuzzy.Filename},
// fuzzy.Symbol
{p: "foo", str: "abc::foo", want: "abc::[foo]", input: fuzzy.Symbol},
{p: "foo", str: "foo.foo", want: "foo.[foo]", input: fuzzy.Symbol},
{p: "foo", str: "fo_oo.o_oo", want: "[fo]_oo.[o]_oo", input: fuzzy.Symbol},
{p: "foo", str: "fo_oo.fo_oo", want: "fo_oo.[fo]_[o]o", input: fuzzy.Symbol},
{p: "fo_o", str: "fo_oo.o_oo", want: "[f]o_oo.[o_o]o", input: fuzzy.Symbol},
{p: "fOO", str: "fo_oo.o_oo", want: "[f]o_oo.[o]_[o]o", input: fuzzy.Symbol},
{p: "tedit", str: "foo.TextEdit", want: "foo.[T]ext[Edit]", input: fuzzy.Symbol},
{p: "TEdit", str: "foo.TextEdit", want: "foo.[T]ext[Edit]", input: fuzzy.Symbol},
{p: "Tedit", str: "foo.TextEdit", want: "foo.[T]ext[Edit]", input: fuzzy.Symbol},
{p: "Tedit", str: "foo.Textedit", want: "foo.[Te]xte[dit]", input: fuzzy.Symbol},
{p: "TEdit", str: "foo.Textedit", want: "", input: fuzzy.Symbol},
{p: "te", str: "foo.Textedit", want: "foo.[Te]xtedit", input: fuzzy.Symbol},
{p: "ee", str: "foo.Textedit", want: "", input: fuzzy.Symbol}, // short middle of the word match
{p: "ex", str: "foo.Textedit", want: "foo.T[ex]tedit", input: fuzzy.Symbol},
{p: "exdi", str: "foo.Textedit", want: "", input: fuzzy.Symbol}, // short middle of the word match
{p: "exdit", str: "foo.Textedit", want: "", input: fuzzy.Symbol}, // short middle of the word match
{p: "extdit", str: "foo.Textedit", want: "foo.T[ext]e[dit]", input: fuzzy.Symbol},
{p: "e", str: "foo.Textedit", want: "foo.T[e]xtedit", input: fuzzy.Symbol},
{p: "E", str: "foo.Textedit", want: "foo.T[e]xtedit", input: fuzzy.Symbol},
{p: "ed", str: "foo.Textedit", want: "foo.Text[ed]it", input: fuzzy.Symbol},
{p: "edt", str: "foo.Textedit", want: "", input: fuzzy.Symbol}, // short middle of the word match
{p: "edit", str: "foo.Textedit", want: "foo.Text[edit]", input: fuzzy.Symbol},
{p: "edin", str: "foo.TexteditNum", want: "foo.Text[edi]t[N]um", input: fuzzy.Symbol},
{p: "n", str: "node.GoNodeMax", want: "[n]ode.GoNodeMax", input: fuzzy.Symbol},
{p: "N", str: "node.GoNodeMax", want: "[n]ode.GoNodeMax", input: fuzzy.Symbol},
{p: "completio", str: "completion", want: "[completio]n", input: fuzzy.Symbol},
{p: "completio", str: "completion.None", want: "[completio]n.None", input: fuzzy.Symbol},
{p: "foo", str: "abc::foo", want: "abc::[foo]"},
{p: "foo", str: "foo.foo", want: "foo.[foo]"},
{p: "foo", str: "fo_oo.o_oo", want: "[fo]_oo.[o]_oo"},
{p: "foo", str: "fo_oo.fo_oo", want: "fo_oo.[fo]_[o]o"},
{p: "fo_o", str: "fo_oo.o_oo", want: "[f]o_oo.[o_o]o"},
{p: "fOO", str: "fo_oo.o_oo", want: "[f]o_oo.[o]_[o]o"},
{p: "tedit", str: "foo.TextEdit", want: "foo.[T]ext[Edit]"},
{p: "TEdit", str: "foo.TextEdit", want: "foo.[T]ext[Edit]"},
{p: "Tedit", str: "foo.TextEdit", want: "foo.[T]ext[Edit]"},
{p: "Tedit", str: "foo.Textedit", want: "foo.[Te]xte[dit]"},
{p: "TEdit", str: "foo.Textedit", want: ""},
{p: "te", str: "foo.Textedit", want: "foo.[Te]xtedit"},
{p: "ee", str: "foo.Textedit", want: ""}, // short middle of the word match
{p: "ex", str: "foo.Textedit", want: "foo.T[ex]tedit"},
{p: "exdi", str: "foo.Textedit", want: ""}, // short middle of the word match
{p: "exdit", str: "foo.Textedit", want: ""}, // short middle of the word match
{p: "extdit", str: "foo.Textedit", want: "foo.T[ext]e[dit]"},
{p: "e", str: "foo.Textedit", want: "foo.T[e]xtedit"},
{p: "E", str: "foo.Textedit", want: "foo.T[e]xtedit"},
{p: "ed", str: "foo.Textedit", want: "foo.Text[ed]it"},
{p: "edt", str: "foo.Textedit", want: ""}, // short middle of the word match
{p: "edit", str: "foo.Textedit", want: "foo.Text[edit]"},
{p: "edin", str: "foo.TexteditNum", want: "foo.Text[edi]t[N]um"},
{p: "n", str: "node.GoNodeMax", want: "[n]ode.GoNodeMax"},
{p: "N", str: "node.GoNodeMax", want: "[n]ode.GoNodeMax"},
{p: "completio", str: "completion", want: "[completio]n"},
{p: "completio", str: "completion.None", want: "[completio]n.None"},
}
func TestFuzzyMatcherRanges(t *testing.T) {
for _, tc := range fuzzyMatcherTestCases {
matcher := fuzzy.NewMatcher(tc.p, tc.input)
matcher := fuzzy.NewMatcher(tc.p)
score := matcher.Score(tc.str)
if tc.want == "" {
if score >= 0 {
@ -299,7 +228,7 @@ var scoreTestCases = []struct {
func TestScores(t *testing.T) {
for _, tc := range scoreTestCases {
matcher := fuzzy.NewMatcher(tc.p, fuzzy.Symbol)
matcher := fuzzy.NewMatcher(tc.p)
got := math.Round(float64(matcher.Score(tc.str))*1e5) / 1e5
if got != tc.want {
t.Errorf("Score(%s, %s) = %v, want: %v", tc.p, tc.str, got, tc.want)
@ -336,7 +265,7 @@ func BenchmarkMatcher(b *testing.B) {
"Foo",
}
matcher := fuzzy.NewMatcher(pattern, fuzzy.Text)
matcher := fuzzy.NewMatcher(pattern)
b.ResetTimer()
for i := 0; i < b.N; i++ {

View File

@ -259,7 +259,7 @@ func (c *completer) setSurrounding(ident *ast.Ident) {
}
if c.opts.FuzzyMatching {
c.matcher = fuzzy.NewMatcher(c.surrounding.Prefix(), fuzzy.Symbol)
c.matcher = fuzzy.NewMatcher(c.surrounding.Prefix())
} else if c.opts.CaseSensitive {
c.matcher = prefixMatcher(c.surrounding.Prefix())
} else {

View File

@ -154,7 +154,7 @@ func (r *runner) DeepCompletion(t *testing.T, src span.Span, test tests.Completi
if !strings.Contains(string(src.URI()), "builtins") {
list = tests.FilterBuiltins(list)
}
fuzzyMatcher := fuzzy.NewMatcher(prefix, fuzzy.Symbol)
fuzzyMatcher := fuzzy.NewMatcher(prefix)
var got []protocol.CompletionItem
for _, item := range list {
if fuzzyMatcher.Score(item.Label) < 0 {
@ -182,7 +182,7 @@ func (r *runner) FuzzyCompletion(t *testing.T, src span.Span, test tests.Complet
}
var fuzzyMatcher *fuzzy.Matcher
if prefix != "" {
fuzzyMatcher = fuzzy.NewMatcher(prefix, fuzzy.Symbol)
fuzzyMatcher = fuzzy.NewMatcher(prefix)
}
var got []protocol.CompletionItem
for _, item := range list {
@ -225,7 +225,7 @@ func (r *runner) RankCompletion(t *testing.T, src span.Span, test tests.Completi
if !strings.Contains(string(src.URI()), "builtins") {
list = tests.FilterBuiltins(list)
}
fuzzyMatcher := fuzzy.NewMatcher(prefix, fuzzy.Symbol)
fuzzyMatcher := fuzzy.NewMatcher(prefix)
var got []protocol.CompletionItem
for _, item := range list {
if fuzzyMatcher.Score(item.Label) < 0 {