// Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package collate_test import ( "bytes" "exp/locale/collate" "testing" ) type weightsTest struct { opt opts in, out ColElems } type opts struct { lev int alt collate.AlternateHandling top int backwards bool caseLevel bool } func (o opts) level() collate.Level { if o.lev == 0 { return collate.Quaternary } return collate.Level(o.lev - 1) } func (o opts) collator() *collate.Collator { c := &collate.Collator{ Strength: o.level(), Alternate: o.alt, Backwards: o.backwards, CaseLevel: o.caseLevel, } collate.SetTop(c, o.top) return c } const ( maxQ = 0x1FFFFF ) func wpq(p, q int) collate.Weights { return collate.W(p, defaults.Secondary, defaults.Tertiary, q) } func wsq(s, q int) collate.Weights { return collate.W(0, s, defaults.Tertiary, q) } func wq(q int) collate.Weights { return collate.W(0, 0, 0, q) } var zero = w(0, 0, 0, 0) var processTests = []weightsTest{ // Shifted { // simple sequence of non-variables opt: opts{alt: collate.AltShifted, top: 100}, in: ColElems{w(200), w(300), w(400)}, out: ColElems{wpq(200, maxQ), wpq(300, maxQ), wpq(400, maxQ)}, }, { // first is a variable opt: opts{alt: collate.AltShifted, top: 250}, in: ColElems{w(200), w(300), w(400)}, out: ColElems{wq(200), wpq(300, maxQ), wpq(400, maxQ)}, }, { // all but first are variable opt: opts{alt: collate.AltShifted, top: 999}, in: ColElems{w(1000), w(200), w(300), w(400)}, out: ColElems{wpq(1000, maxQ), wq(200), wq(300), wq(400)}, }, { // first is a modifier opt: opts{alt: collate.AltShifted, top: 999}, in: ColElems{w(0, 10), w(1000)}, out: ColElems{wsq(10, maxQ), wpq(1000, maxQ)}, }, { // primary ignorables opt: opts{alt: collate.AltShifted, top: 250}, in: ColElems{w(200), w(0, 10), w(300), w(0, 15), w(400)}, out: ColElems{wq(200), zero, wpq(300, maxQ), wsq(15, maxQ), wpq(400, maxQ)}, }, { // secondary ignorables opt: opts{alt: collate.AltShifted, top: 250}, in: ColElems{w(200), w(0, 0, 10), w(300), w(0, 0, 15), w(400)}, out: ColElems{wq(200), zero, wpq(300, maxQ), w(0, 0, 15, maxQ), wpq(400, maxQ)}, }, { // tertiary ignorables, no change opt: opts{alt: collate.AltShifted, top: 250}, in: ColElems{w(200), zero, w(300), zero, w(400)}, out: ColElems{wq(200), zero, wpq(300, maxQ), zero, wpq(400, maxQ)}, }, // ShiftTrimmed (same as Shifted) { // simple sequence of non-variables opt: opts{alt: collate.AltShiftTrimmed, top: 100}, in: ColElems{w(200), w(300), w(400)}, out: ColElems{wpq(200, maxQ), wpq(300, maxQ), wpq(400, maxQ)}, }, { // first is a variable opt: opts{alt: collate.AltShiftTrimmed, top: 250}, in: ColElems{w(200), w(300), w(400)}, out: ColElems{wq(200), wpq(300, maxQ), wpq(400, maxQ)}, }, { // all but first are variable opt: opts{alt: collate.AltShiftTrimmed, top: 999}, in: ColElems{w(1000), w(200), w(300), w(400)}, out: ColElems{wpq(1000, maxQ), wq(200), wq(300), wq(400)}, }, { // first is a modifier opt: opts{alt: collate.AltShiftTrimmed, top: 999}, in: ColElems{w(0, 10), w(1000)}, out: ColElems{wsq(10, maxQ), wpq(1000, maxQ)}, }, { // primary ignorables opt: opts{alt: collate.AltShiftTrimmed, top: 250}, in: ColElems{w(200), w(0, 10), w(300), w(0, 15), w(400)}, out: ColElems{wq(200), zero, wpq(300, maxQ), wsq(15, maxQ), wpq(400, maxQ)}, }, { // secondary ignorables opt: opts{alt: collate.AltShiftTrimmed, top: 250}, in: ColElems{w(200), w(0, 0, 10), w(300), w(0, 0, 15), w(400)}, out: ColElems{wq(200), zero, wpq(300, maxQ), w(0, 0, 15, maxQ), wpq(400, maxQ)}, }, { // tertiary ignorables, no change opt: opts{alt: collate.AltShiftTrimmed, top: 250}, in: ColElems{w(200), zero, w(300), zero, w(400)}, out: ColElems{wq(200), zero, wpq(300, maxQ), zero, wpq(400, maxQ)}, }, // Blanked { // simple sequence of non-variables opt: opts{alt: collate.AltBlanked, top: 100}, in: ColElems{w(200), w(300), w(400)}, out: ColElems{w(200), w(300), w(400)}, }, { // first is a variable opt: opts{alt: collate.AltBlanked, top: 250}, in: ColElems{w(200), w(300), w(400)}, out: ColElems{zero, w(300), w(400)}, }, { // all but first are variable opt: opts{alt: collate.AltBlanked, top: 999}, in: ColElems{w(1000), w(200), w(300), w(400)}, out: ColElems{w(1000), zero, zero, zero}, }, { // first is a modifier opt: opts{alt: collate.AltBlanked, top: 999}, in: ColElems{w(0, 10), w(1000)}, out: ColElems{w(0, 10), w(1000)}, }, { // primary ignorables opt: opts{alt: collate.AltBlanked, top: 250}, in: ColElems{w(200), w(0, 10), w(300), w(0, 15), w(400)}, out: ColElems{zero, zero, w(300), w(0, 15), w(400)}, }, { // secondary ignorables opt: opts{alt: collate.AltBlanked, top: 250}, in: ColElems{w(200), w(0, 0, 10), w(300), w(0, 0, 15), w(400)}, out: ColElems{zero, zero, w(300), w(0, 0, 15), w(400)}, }, { // tertiary ignorables, no change opt: opts{alt: collate.AltBlanked, top: 250}, in: ColElems{w(200), zero, w(300), zero, w(400)}, out: ColElems{zero, zero, w(300), zero, w(400)}, }, // Non-ignorable: input is always equal to output. { // all but first are variable opt: opts{alt: collate.AltNonIgnorable, top: 999}, in: ColElems{w(1000), w(200), w(300), w(400)}, out: ColElems{w(1000), w(200), w(300), w(400)}, }, { // primary ignorables opt: opts{alt: collate.AltNonIgnorable, top: 250}, in: ColElems{w(200), w(0, 10), w(300), w(0, 15), w(400)}, out: ColElems{w(200), w(0, 10), w(300), w(0, 15), w(400)}, }, { // secondary ignorables opt: opts{alt: collate.AltNonIgnorable, top: 250}, in: ColElems{w(200), w(0, 0, 10), w(300), w(0, 0, 15), w(400)}, out: ColElems{w(200), w(0, 0, 10), w(300), w(0, 0, 15), w(400)}, }, { // tertiary ignorables, no change opt: opts{alt: collate.AltNonIgnorable, top: 250}, in: ColElems{w(200), zero, w(300), zero, w(400)}, out: ColElems{w(200), zero, w(300), zero, w(400)}, }, } func TestProcessWeights(t *testing.T) { for i, tt := range processTests { res := collate.ProcessWeights(tt.opt.alt, tt.opt.top, tt.in) if len(res) != len(tt.out) { t.Errorf("%d: len(ws) was %d; want %d (%v should be %v)", i, len(res), len(tt.out), res, tt.out) continue } for j, w := range res { if w != tt.out[j] { t.Errorf("%d: Weights %d was %v; want %v", i, j, w, tt.out[j]) } } } } type keyFromElemTest struct { opt opts in ColElems out []byte } var defS = byte(defaults.Secondary) var defT = byte(defaults.Tertiary) const sep = 0 // separator byte var keyFromElemTests = []keyFromElemTest{ { // simple primary and secondary weights. opts{alt: collate.AltShifted}, ColElems{w(0x200), w(0x7FFF), w(0, 0x30), w(0x100)}, []byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary sep, sep, defT, defT, defT, defT, // tertiary sep, 0xFF, 0xFF, 0xFF, 0xFF, // quaternary }, }, { // same as first, but with zero element that need to be removed opts{alt: collate.AltShifted}, ColElems{w(0x200), zero, w(0x7FFF), w(0, 0x30), zero, w(0x100)}, []byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary sep, sep, defT, defT, defT, defT, // tertiary sep, 0xFF, 0xFF, 0xFF, 0xFF, // quaternary }, }, { // same as first, with large primary values opts{alt: collate.AltShifted}, ColElems{w(0x200), w(0x8000), w(0, 0x30), w(0x12345)}, []byte{0x2, 0, 0x80, 0x80, 0x00, 0x81, 0x23, 0x45, // primary sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary sep, sep, defT, defT, defT, defT, // tertiary sep, 0xFF, 0xFF, 0xFF, 0xFF, // quaternary }, }, { // same as first, but with the secondary level backwards opts{alt: collate.AltShifted, backwards: true}, ColElems{w(0x200), w(0x7FFF), w(0, 0x30), w(0x100)}, []byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary sep, sep, 0, defS, 0, 0x30, 0, defS, 0, defS, // secondary sep, sep, defT, defT, defT, defT, // tertiary sep, 0xFF, 0xFF, 0xFF, 0xFF, // quaternary }, }, { // same as first, ignoring quaternary level opts{alt: collate.AltShifted, lev: 3}, ColElems{w(0x200), zero, w(0x7FFF), w(0, 0x30), zero, w(0x100)}, []byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary sep, sep, defT, defT, defT, defT, // tertiary }, }, { // same as first, ignoring tertiary level opts{alt: collate.AltShifted, lev: 2}, ColElems{w(0x200), zero, w(0x7FFF), w(0, 0x30), zero, w(0x100)}, []byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary }, }, { // same as first, ignoring secondary level opts{alt: collate.AltShifted, lev: 1}, ColElems{w(0x200), zero, w(0x7FFF), w(0, 0x30), zero, w(0x100)}, []byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00}, }, { // simple primary and secondary weights. opts{alt: collate.AltShiftTrimmed, top: 0x250}, ColElems{w(0x300), w(0x200), w(0x7FFF), w(0, 0x30), w(0x800)}, []byte{0x3, 0, 0x7F, 0xFF, 0x8, 0x00, // primary sep, sep, 0, defS, 0, defS, 0, 0x30, 0, defS, // secondary sep, sep, defT, defT, defT, defT, // tertiary sep, 0xFF, 0x2, 0, // quaternary }, }, { // as first, primary with case level enabled opts{alt: collate.AltShifted, lev: 1, caseLevel: true}, ColElems{w(0x200), w(0x7FFF), w(0, 0x30), w(0x100)}, []byte{0x2, 0, 0x7F, 0xFF, 0x1, 0x00, // primary sep, sep, // secondary sep, sep, defT, defT, defT, defT, // tertiary }, }, } func TestKeyFromElems(t *testing.T) { buf := collate.Buffer{} for i, tt := range keyFromElemTests { buf.Reset() ws := collate.ProcessWeights(tt.opt.alt, tt.opt.top, tt.in) res := collate.KeyFromElems(tt.opt.collator(), &buf, ws) if len(res) != len(tt.out) { t.Errorf("%d: len(ws) was %d; want %d (%X should be %X)", i, len(res), len(tt.out), res, tt.out) } n := len(res) if len(tt.out) < n { n = len(tt.out) } for j, c := range res[:n] { if c != tt.out[j] { t.Errorf("%d: byte %d was %X; want %X", i, j, c, tt.out[j]) } } } } func TestGetColElems(t *testing.T) { for i, tt := range appendNextTests { c, err := makeTable(tt.in) if err != nil { // error is reported in TestAppendNext continue } // Create one large test per table str := make([]byte, 0, 4000) out := ColElems{} for len(str) < 3000 { for _, chk := range tt.chk { str = append(str, chk.in[:chk.n]...) out = append(out, chk.out...) } } for j, chk := range append(tt.chk, check{string(str), len(str), out}) { ws := collate.GetColElems(c, []byte(chk.in)[:chk.n]) if len(ws) != len(chk.out) { t.Errorf("%d:%d: len(ws) was %d; want %d", i, j, len(ws), len(chk.out)) continue } cnt := 0 for k, w := range ws { if w != chk.out[k] { t.Errorf("%d:%d: Weights %d was %v; want %v", i, j, k, w, chk.out[k]) cnt++ } if cnt > 10 { break } } } } } type keyTest struct { in string out []byte } var keyTests = []keyTest{ {"abc", []byte{0, 100, 0, 200, 1, 44, 0, 0, 0, 32, 0, 32, 0, 32, 0, 0, 2, 2, 2, 0, 255, 255, 255}, }, {"a\u0301", []byte{0, 102, 0, 0, 0, 32, 0, 0, 2, 0, 255}, }, {"aaaaa", []byte{0, 100, 0, 100, 0, 100, 0, 100, 0, 100, 0, 0, 0, 32, 0, 32, 0, 32, 0, 32, 0, 32, 0, 0, 2, 2, 2, 2, 2, 0, 255, 255, 255, 255, 255, }, }, } func TestKey(t *testing.T) { c, _ := makeTable(appendNextTests[4].in) c.Alternate = collate.AltShifted c.Strength = collate.Quaternary buf := collate.Buffer{} keys1 := [][]byte{} keys2 := [][]byte{} for _, tt := range keyTests { keys1 = append(keys1, c.Key(&buf, []byte(tt.in))) keys2 = append(keys2, c.KeyFromString(&buf, tt.in)) } // Separate generation from testing to ensure buffers are not overwritten. for i, tt := range keyTests { if !bytes.Equal(keys1[i], tt.out) { t.Errorf("%d: Key(%q) = %d; want %d", i, tt.in, keys1[i], tt.out) } if !bytes.Equal(keys2[i], tt.out) { t.Errorf("%d: KeyFromString(%q) = %d; want %d", i, tt.in, keys2[i], tt.out) } } } type compareTest struct { a, b string res int // comparison result } var compareTests = []compareTest{ {"a\u0301", "a", 1}, {"a\u0301b", "ab", 1}, {"a", "a\u0301", -1}, {"ab", "a\u0301b", -1}, {"bc", "a\u0301c", 1}, {"ab", "aB", -1}, {"a\u0301", "a\u0301", 0}, {"a", "a", 0}, // Only clip prefixes of whole runes. {"\u302E", "\u302F", 1}, // Don't clip prefixes when last rune of prefix may be part of contraction. {"a\u035E", "a\u0301\u035F", -1}, {"a\u0301\u035Fb", "a\u0301\u035F", -1}, } func TestCompare(t *testing.T) { c, _ := makeTable(appendNextTests[4].in) for i, tt := range compareTests { if res := c.Compare([]byte(tt.a), []byte(tt.b)); res != tt.res { t.Errorf("%d: Compare(%q, %q) == %d; want %d", i, tt.a, tt.b, res, tt.res) } if res := c.CompareString(tt.a, tt.b); res != tt.res { t.Errorf("%d: CompareString(%q, %q) == %d; want %d", i, tt.a, tt.b, res, tt.res) } } }