qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-25 11:27:56 -07:00
Code Releases Activity

exp/locale/collate: add code to ignore tests with (unpaired) surrogates.

Browse Source 
In the regtest data, surrogates are assigned primary weights based on
the surrogate code point value.  Go now converts surrogates to FFFD, however,
meaning that the primary weight is based on this code point instead.
This change drops tests with surrogates and lets the tests pass.

R=r
CC=golang-dev
https://golang.org/cl/6461100
This commit is contained in:
Marcel van Lohuizen 2012-08-24 15:56:07 +02:00
parent 75af013229
commit c61a185f35
1 changed files with 18 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/pkg/exp/locale/collate/regtest.go
View File

@ -24,6 +24,7 @@ import (
"strconv" "strconv"
"strings" "strings"
"unicode" "unicode"
"unicode/utf8"
) )
// This regression test runs tests for the test files in CollationTest.zip // This regression test runs tests for the test files in CollationTest.zip
@ -53,7 +54,7 @@ var localFiles = flag.Bool("local",
type Test struct { type Test struct {
name string name string
str []string str [][]byte
comment []string comment []string
} }
@ -186,15 +187,24 @@ func loadTestData() []Test {
if m == nil || len(m) < 3 { if m == nil || len(m) < 3 {
log.Fatalf(`Failed to parse: "%s" result: %#v`, line, m) log.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
} }
str := "" str := []byte{}
// In the regression test data (unpaired) surrogates are assigned a weight
// corresponding to their code point value. However, utf8.DecodeRune,
// which is used to compute the implicit weight, assigns FFFD to surrogates.
// We therefore skip tests with surrogates. This skips about 35 entries
// per test.
valid := true
for _, split := range strings.Split(m[1], " ") { for _, split := range strings.Split(m[1], " ") {
r, err := strconv.ParseUint(split, 16, 64) r, err := strconv.ParseUint(split, 16, 64)
Error(err) Error(err)
str += string(rune(r)) valid = valid && utf8.ValidRune(rune(r))
str = append(str, string(rune(r))...)
} }
if valid {
test.str = append(test.str, str) test.str = append(test.str, str)
test.comment = append(test.comment, m[2]) test.comment = append(test.comment, m[2])
} }
}
tests = append(tests, test) tests = append(tests, test)
} }
return tests return tests
@ -227,13 +237,13 @@ func doTest(t Test) {
c.Alternate = collate.AltNonIgnorable c.Alternate = collate.AltNonIgnorable
} }
prev := []byte(t.str[0]) prev := t.str[0]
for i := 1; i < len(t.str); i++ { for i := 1; i < len(t.str); i++ {
s := []byte(t.str[i]) s := t.str[i]
ka := c.Key(b, prev) ka := c.Key(b, prev)
kb := c.Key(b, s) kb := c.Key(b, s)
if r := bytes.Compare(ka, kb); r == 1 { if r := bytes.Compare(ka, kb); r == 1 {
fail(t, "%d: Key(%.4X) < Key(%.4X) (%X < %X) == %d; want -1 or 0", i, runes(prev), runes(s), ka, kb, r) fail(t, "%d: Key(%.4X) < Key(%.4X) (%X < %X) == %d; want -1 or 0", i, []rune(string(prev)), []rune(string(s)), ka, kb, r)
prev = s prev = s
continue continue
} }