1
0
mirror of https://github.com/golang/go synced 2024-11-12 04:50:21 -07:00

exp/locale/collate: added indices to builder for reusing blocks between locales.

Refactored build + buildTrie into build + buildOrdering.
Note that since the tailoring code is not checked in yet, all tailorings are identical
to root.  The table therefore should not and does not grow at this point.

R=r
CC=golang-dev
https://golang.org/cl/6500087
This commit is contained in:
Marcel van Lohuizen 2012-09-08 10:46:55 +09:00
parent 21d94a22fe
commit ef48dfa310
3 changed files with 79 additions and 59 deletions

View File

@ -42,6 +42,11 @@ type Builder struct {
minNonVar int // lowest primary recorded for a variable
varTop int // highest primary recorded for a non-variable
// indexes used for reusing expansions and contractions
expIndex map[string]int // positions of expansions keyed by their string representation
ctHandle map[string]ctHandle // contraction handles keyed by a concatenation of the suffixes
ctElem map[string]int // contraction elements keyed by their string representation
}
// A Tailoring builds a collation table based on another collation table.
@ -50,24 +55,30 @@ type Builder struct {
// collation tables. The CLDR contains pre-defined tailorings for a variety
// of languages (See http://www.unicode.org/Public/cldr/2.0.1/core.zip.)
type Tailoring struct {
id string
id string
builder *Builder
index *ordering
// TODO: implement.
}
// NewBuilder returns a new Builder.
func NewBuilder() *Builder {
b := &Builder{
index: newTrieBuilder(),
root: makeRootOrdering(),
return &Builder{
index: newTrieBuilder(),
root: makeRootOrdering(),
expIndex: make(map[string]int),
ctHandle: make(map[string]ctHandle),
ctElem: make(map[string]int),
}
return b
}
// Tailoring returns a Tailoring for the given locale. One should
// have completed all calls to Add before calling Tailoring.
func (b *Builder) Tailoring(locale string) *Tailoring {
t := &Tailoring{
id: locale,
id: locale,
builder: b,
index: b.root.clone(),
}
b.locale = append(b.locale, t)
return t
@ -194,24 +205,45 @@ func (b *Builder) error(e error) {
}
}
func (b *Builder) build() (*table, error) {
if !b.built {
b.built = true
b.t = &table{
maxContractLen: utf8.UTFMax,
variableTop: uint32(b.varTop),
}
func (b *Builder) buildOrdering(o *ordering) {
o.sort()
simplify(o)
b.processExpansions(o) // requires simplify
b.processContractions(o) // requires simplify
b.root.sort()
b.simplify()
b.processExpansions() // requires simplify
b.processContractions() // requires simplify
b.buildTrie() // requires process*
t := newNode()
for e := o.front(); e != nil; e, _ = e.nextIndexed() {
if !e.skip() {
ce, err := e.encode()
b.error(err)
t.insert(e.runes[0], ce)
}
}
if b.err != nil {
return nil, b.err
o.handle = b.index.addTrie(t)
}
func (b *Builder) build() (*table, error) {
if b.built {
return b.t, b.err
}
return b.t, nil
b.built = true
b.t = &table{
maxContractLen: utf8.UTFMax,
variableTop: uint32(b.varTop),
}
b.buildOrdering(&b.root)
b.t.root = b.root.handle
for _, t := range b.locale {
b.buildOrdering(t.index)
if b.err != nil {
break
}
}
i, err := b.index.generate()
b.t.index = *i
b.error(err)
return b.t, b.err
}
// Build builds the root Collator.
@ -263,12 +295,10 @@ func reproducibleFromNFKD(e *entry, exp, nfkd [][]int) bool {
return true
}
func (b *Builder) simplify() {
func simplify(o *ordering) {
// Runes that are a starter of a contraction should not be removed.
// (To date, there is only Kannada character 0CCA.)
keep := make(map[rune]bool)
o := b.root
for e := o.front(); e != nil; e, _ = e.nextIndexed() {
if len(e.runes) > 1 {
keep[e.runes[0]] = true
@ -320,27 +350,24 @@ func (b *Builder) appendExpansion(e *entry) int {
// processExpansions extracts data necessary to generate
// the extraction tables.
func (b *Builder) processExpansions() {
eidx := make(map[string]int)
o := b.root
func (b *Builder) processExpansions(o *ordering) {
for e := o.front(); e != nil; e, _ = e.nextIndexed() {
if !e.expansion() {
continue
}
key := fmt.Sprintf("%v", e.elems)
i, ok := eidx[key]
i, ok := b.expIndex[key]
if !ok {
i = b.appendExpansion(e)
eidx[key] = i
b.expIndex[key] = i
}
e.expansionIndex = i
}
}
func (b *Builder) processContractions() {
func (b *Builder) processContractions(o *ordering) {
// Collate contractions per starter rune.
starters := []rune{}
o := b.root
cm := make(map[rune][]*entry)
for e := o.front(); e != nil; e, _ = e.nextIndexed() {
if e.contraction() {
@ -365,7 +392,6 @@ func (b *Builder) processContractions() {
}
// Build the tries for the contractions.
t := b.t
handlemap := make(map[string]ctHandle)
for _, r := range starters {
l := cm[r]
// Compute suffix strings. There are 31 different contraction suffix
@ -387,14 +413,14 @@ func (b *Builder) processContractions() {
// Unique the suffix set.
sort.Strings(sufx)
key := strings.Join(sufx, "\n")
handle, ok := handlemap[key]
handle, ok := b.ctHandle[key]
if !ok {
var err error
handle, err = t.contractTries.appendTrie(sufx)
if err != nil {
b.error(err)
}
handlemap[key] = handle
b.ctHandle[key] = handle
}
// Bucket sort entries in index order.
es := make([]*entry, len(l))
@ -412,30 +438,22 @@ func (b *Builder) processContractions() {
}
es[o] = e
}
// Store info in entry for starter rune.
es[0].contractionIndex = len(t.contractElem)
es[0].contractionHandle = handle
// Add collation elements for contractions.
// Create collation elements for contractions.
elems := []uint32{}
for _, e := range es {
ce, err := e.encodeBase()
b.error(err)
t.contractElem = append(t.contractElem, ce)
elems = append(elems, ce)
}
key = fmt.Sprintf("%v", elems)
i, ok := b.ctElem[key]
if !ok {
i = len(t.contractElem)
b.ctElem[key] = i
t.contractElem = append(t.contractElem, elems...)
}
// Store info in entry for starter rune.
es[0].contractionIndex = i
es[0].contractionHandle = handle
}
}
func (b *Builder) buildTrie() {
t := newNode()
o := b.root
for e := o.front(); e != nil; e, _ = e.nextIndexed() {
if !e.skip() {
ce, err := e.encode()
b.error(err)
t.insert(e.runes[0], ce)
}
}
b.t.root = b.index.addTrie(t)
i, err := b.index.generate()
b.t.index = *i
b.error(err)
}

View File

@ -143,8 +143,8 @@ var simplifyMarked = strArray{"\u01C5"}
func TestSimplify(t *testing.T) {
b := newBuilder(t, simplifyTest)
o := b.root
b.simplify()
o := &b.root
simplify(o)
for i, tt := range simplifyTest {
if simplifyRemoved.contains(tt.str) {
@ -186,7 +186,7 @@ func TestExpand(t *testing.T) {
)
b := newBuilder(t, expandTest)
o := &b.root
b.processExpansions()
b.processExpansions(o)
e := o.front()
for _, tt := range expandTest {
@ -234,7 +234,7 @@ func TestContract(t *testing.T) {
)
b := newBuilder(t, contractTest)
o := &b.root
b.processContractions()
b.processContractions(o)
indexMap := make(map[int]bool)
handleMap := make(map[rune]*entry)

View File

@ -180,6 +180,7 @@ func (s sortedEntries) Less(i, j int) bool {
type ordering struct {
entryMap map[string]*entry
ordered []*entry
handle *trieHandle
}
// insert inserts e into both entryMap and ordered.
@ -264,6 +265,7 @@ func (o *ordering) clone() *ordering {
str: e.str,
decompose: e.decompose,
exclude: e.exclude,
logical: e.logical,
}
oo.insert(ne)
}