qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-22 02:44:39 -07:00
Code Releases Activity

unicode: make the tables smaller.

Browse Source 
By splitting the ranges into 16-bit values and 32-bit values,
we can reduce about 3000 entries by 48 bits per entry, or about
16KB, at the cost of a little more complexity in the code.

R=iant, bradfitz, rsc, r
CC=golang-dev
https://golang.org/cl/4547066
This commit is contained in:
Rob Pike 2011-05-31 09:58:07 +10:00
parent 2c6a2a9773
commit 0de328edd6
4 changed files with 4481 additions and 3969 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
src/pkg/unicode/letter.go
View File

@ -11,13 +11,30 @@ const (
ReplacementChar = 0xFFFD // Represents invalid code points. ReplacementChar = 0xFFFD // Represents invalid code points.
) )
// RangeTable defines a set of Unicode code points by listing the ranges of
// code points within the set. The ranges are listed in two slices
// to save space: a slice of 16-bit ranges and a slice of 32-bit ranges.
// The two slices must be in sorted order and non-overlapping.
type RangeTable struct {
R16 []Range16
R32 []Range32
}
// The representation of a range of Unicode code points. The range runs from Lo to Hi // Range16 represents of a range of 16-bit Unicode code points. The range runs from Lo to Hi
// inclusive and has the specified stride. // inclusive and has the specified stride.
type Range struct { type Range16 struct {
Lo int Lo uint16
Hi int Hi uint16
Stride int Stride uint16
}
// Range32 represents of a range of Unicode code points and is used when one or
// more of the values will not fit in 16 bits. The range runs from Lo to Hi
// inclusive and has the specified stride.
type Range32 struct {
Lo uint32
Hi uint32
Stride uint32
} }
// CaseRange represents a range of Unicode code points for simple (one // CaseRange represents a range of Unicode code points for simple (one
@ -60,22 +77,8 @@ const (
UpperLower = MaxRune + 1 // (Cannot be a valid delta.) UpperLower = MaxRune + 1 // (Cannot be a valid delta.)
) )
// Is tests whether rune is in the specified table of ranges. // is16 uses binary search to test whether rune is in the specified slice of 16-bit ranges.
func Is(ranges []Range, rune int) bool { func is16(ranges []Range16, rune uint16) bool {
// common case: rune is ASCII or Latin-1
if rune < 0x100 {
for _, r := range ranges {
if rune > r.Hi {
continue
}
if rune < r.Lo {
return false
}
return (rune-r.Lo)%r.Stride == 0
}
return false
}
// binary search over ranges // binary search over ranges
lo := 0 lo := 0
hi := len(ranges) hi := len(ranges)
@ -94,6 +97,63 @@ func Is(ranges []Range, rune int) bool {
return false return false
} }
// is32 uses binary search to test whether rune is in the specified slice of 32-bit ranges.
func is32(ranges []Range32, rune uint32) bool {
// binary search over ranges
lo := 0
hi := len(ranges)
for lo < hi {
m := lo + (hi-lo)/2
r := ranges[m]
if r.Lo <= rune && rune <= r.Hi {
return (rune-r.Lo)%r.Stride == 0
}
if rune < r.Lo {
hi = m
} else {
lo = m + 1
}
}
return false
}
// Is tests whether rune is in the specified table of ranges.
func Is(rangeTab *RangeTable, rune int) bool {
// common case: rune is ASCII or Latin-1.
if rune < 0x100 {
r16 := uint16(rune)
for _, r := range rangeTab.R16 {
if r16 > r.Hi {
continue
}
if r16 < r.Lo {
return false
}
return (r16-r.Lo)%r.Stride == 0
}
r32 := uint32(rune)
for _, r := range rangeTab.R32 {
if r32 > r.Hi {
continue
}
if r32 < r.Lo {
return false
}
return (r32-r.Lo)%r.Stride == 0
}
return false
}
r16 := rangeTab.R16
if len(r16) > 0 && rune <= int(r16[len(r16)-1].Hi) {
return is16(r16, uint16(rune))
}
r32 := rangeTab.R32
if len(r32) > 0 && rune >= int(r32[0].Lo) {
return is32(r32, uint32(rune))
}
return false
}
// IsUpper reports whether the rune is an upper case letter. // IsUpper reports whether the rune is an upper case letter.
func IsUpper(rune int) bool { func IsUpper(rune int) bool {
if rune < 0x80 { // quick ASCII check if rune < 0x80 { // quick ASCII check

97
src/pkg/unicode/maketables.go
View File

@ -28,6 +28,7 @@ func main() {
printScriptOrProperty(false) printScriptOrProperty(false)
printScriptOrProperty(true) printScriptOrProperty(true)
printCases() printCases()
printSizes()
} }
var dataURL = flag.String("data", "", "full URL for UnicodeData.txt; defaults to --url/UnicodeData.txt") var dataURL = flag.String("data", "", "full URL for UnicodeData.txt; defaults to --url/UnicodeData.txt")
@ -278,16 +279,16 @@ func loadChars() {
switch parseCategory(line[0 : len(line)-1]) { switch parseCategory(line[0 : len(line)-1]) {
case SNormal: case SNormal:
if first != 0 { if first != 0 {
logger.Fatalf("bad state normal at U+%04X", lastChar) logger.Fatalf("bad state normal at %U", lastChar)
} }
case SFirst: case SFirst:
if first != 0 { if first != 0 {
logger.Fatalf("bad state first at U+%04X", lastChar) logger.Fatalf("bad state first at %U", lastChar)
} }
first = lastChar first = lastChar
case SLast: case SLast:
if first == 0 { if first == 0 {
logger.Fatalf("bad state last at U+%04X", lastChar) logger.Fatalf("bad state last at %U", lastChar)
} }
for i := first + 1; i <= lastChar; i++ { for i := first + 1; i <= lastChar; i++ {
chars[i] = chars[first] chars[i] = chars[first]
@ -299,6 +300,15 @@ func loadChars() {
resp.Body.Close() resp.Body.Close()
} }
const progHeader = `// Generated by running
// maketables --tables=%s --data=%s
// DO NOT EDIT
package unicode
`
func printCategories() { func printCategories() {
if *tablelist == "" { if *tablelist == "" {
return return
@ -312,20 +322,14 @@ func printCategories() {
fullCategoryTest(list) fullCategoryTest(list)
return return
} }
fmt.Printf( fmt.Printf(progHeader, *tablelist, *dataURL)
"// Generated by running\n"+
"// maketables --tables=%s --data=%s\n"+
"// DO NOT EDIT\n\n"+
"package unicode\n\n",
*tablelist,
*dataURL)
fmt.Println("// Version is the Unicode edition from which the tables are derived.") fmt.Println("// Version is the Unicode edition from which the tables are derived.")
fmt.Printf("const Version = %q\n\n", version()) fmt.Printf("const Version = %q\n\n", version())
if *tablelist == "all" { if *tablelist == "all" {
fmt.Println("// Categories is the set of Unicode data tables.") fmt.Println("// Categories is the set of Unicode data tables.")
fmt.Println("var Categories = map[string] []Range {") fmt.Println("var Categories = map[string] *RangeTable {")
for k := range category { for k := range category {
fmt.Printf("\t%q: %s,\n", k, k) fmt.Printf("\t%q: %s,\n", k, k)
} }
@ -364,12 +368,12 @@ func printCategories() {
ndecl++ ndecl++
if name == "letter" { // special case if name == "letter" { // special case
dumpRange( dumpRange(
"var letter = []Range {\n", "var letter = &RangeTable{\n",
letterOp) letterOp)
continue continue
} }
dumpRange( dumpRange(
fmt.Sprintf("var _%s = []Range {\n", name), fmt.Sprintf("var _%s = &RangeTable{\n", name),
func(code int) bool { return chars[code].category == name }) func(code int) bool { return chars[code].category == name })
} }
decl.Sort() decl.Sort()
@ -382,12 +386,15 @@ func printCategories() {
type Op func(code int) bool type Op func(code int) bool
const format = "\t{0x%04x, 0x%04x, %d},\n" const format = "\t\t{0x%04x, 0x%04x, %d},\n"
func dumpRange(header string, inCategory Op) { func dumpRange(header string, inCategory Op) {
fmt.Print(header) fmt.Print(header)
next := 0 next := 0
fmt.Print("\tR16: []Range16{\n")
// one Range for each iteration // one Range for each iteration
count := &range16Count
size := 16
for { for {
// look for start of range // look for start of range
for next < len(chars) && !inCategory(next) { for next < len(chars) && !inCategory(next) {
@ -427,10 +434,18 @@ func dumpRange(header string, inCategory Op) {
break break
} }
} }
if size == 16 && (lo >= 1<<16 || hi >= 1<<16) {
fmt.Print("\t},\n")
fmt.Print("\tR32: []Range32{\n")
size = 32
count = &range32Count
}
fmt.Printf(format, lo, hi, stride) fmt.Printf(format, lo, hi, stride)
*count++
// next range: start looking where this range ends // next range: start looking where this range ends
next = hi + 1 next = hi + 1
} }
fmt.Print("\t},\n")
fmt.Print("}\n\n") fmt.Print("}\n\n")
} }
@ -454,12 +469,12 @@ func fullCategoryTest(list []string) {
} }
} }
func verifyRange(name string, inCategory Op, table []unicode.Range) { func verifyRange(name string, inCategory Op, table *unicode.RangeTable) {
for i := range chars { for i := range chars {
web := inCategory(i) web := inCategory(i)
pkg := unicode.Is(table, i) pkg := unicode.Is(table, i)
if web != pkg { if web != pkg {
fmt.Fprintf(os.Stderr, "%s: U+%04X: web=%t pkg=%t\n", name, i, web, pkg) fmt.Fprintf(os.Stderr, "%s: %U: web=%t pkg=%t\n", name, i, web, pkg)
} }
} }
} }
@ -497,22 +512,22 @@ func parseScript(line string, scripts map[string][]Script) {
} }
// The script tables have a lot of adjacent elements. Fold them together. // The script tables have a lot of adjacent elements. Fold them together.
func foldAdjacent(r []Script) []unicode.Range { func foldAdjacent(r []Script) []unicode.Range32 {
s := make([]unicode.Range, 0, len(r)) s := make([]unicode.Range32, 0, len(r))
j := 0 j := 0
for i := 0; i < len(r); i++ { for i := 0; i < len(r); i++ {
if j > 0 && int(r[i].lo) == s[j-1].Hi+1 { if j > 0 && r[i].lo == s[j-1].Hi+1 {
s[j-1].Hi = int(r[i].hi) s[j-1].Hi = r[i].hi
} else { } else {
s = s[0 : j+1] s = s[0 : j+1]
s[j] = unicode.Range{int(r[i].lo), int(r[i].hi), 1} s[j] = unicode.Range32{uint32(r[i].lo), uint32(r[i].hi), 1}
j++ j++
} }
} }
return s return s
} }
func fullScriptTest(list []string, installed map[string][]unicode.Range, scripts map[string][]Script) { func fullScriptTest(list []string, installed map[string]*unicode.RangeTable, scripts map[string][]Script) {
for _, name := range list { for _, name := range list {
if _, ok := scripts[name]; !ok { if _, ok := scripts[name]; !ok {
logger.Fatal("unknown script", name) logger.Fatal("unknown script", name)
@ -524,7 +539,7 @@ func fullScriptTest(list []string, installed map[string][]unicode.Range, scripts
for _, script := range scripts[name] { for _, script := range scripts[name] {
for r := script.lo; r <= script.hi; r++ { for r := script.lo; r <= script.hi; r++ {
if !unicode.Is(installed[name], int(r)) { if !unicode.Is(installed[name], int(r)) {
fmt.Fprintf(os.Stderr, "U+%04X: not in script %s\n", r, name) fmt.Fprintf(os.Stderr, "%U: not in script %s\n", r, name)
} }
} }
} }
@ -589,10 +604,10 @@ func printScriptOrProperty(doProps bool) {
if flaglist == "all" { if flaglist == "all" {
if doProps { if doProps {
fmt.Println("// Properties is the set of Unicode property tables.") fmt.Println("// Properties is the set of Unicode property tables.")
fmt.Println("var Properties = map[string] []Range {") fmt.Println("var Properties = map[string] *RangeTable{")
} else { } else {
fmt.Println("// Scripts is the set of Unicode script tables.") fmt.Println("// Scripts is the set of Unicode script tables.")
fmt.Println("var Scripts = map[string] []Range {") fmt.Println("var Scripts = map[string] *RangeTable{")
} }
for k := range table { for k := range table {
fmt.Printf("\t%q: %s,\n", k, k) fmt.Printf("\t%q: %s,\n", k, k)
@ -613,11 +628,22 @@ func printScriptOrProperty(doProps bool) {
name, name, name, name) name, name, name, name)
} }
ndecl++ ndecl++
fmt.Printf("var _%s = []Range {\n", name) fmt.Printf("var _%s = &RangeTable {\n", name)
fmt.Print("\tR16: []Range16{\n")
ranges := foldAdjacent(table[name]) ranges := foldAdjacent(table[name])
size := 16
count := &range16Count
for _, s := range ranges { for _, s := range ranges {
if size == 16 && (s.Lo >= 1<<16 || s.Hi >= 1<<16) {
fmt.Print("\t},\n")
fmt.Print("\tR32: []Range32{\n")
size = 32
count = &range32Count
}
*count++
fmt.Printf(format, s.Lo, s.Hi, s.Stride) fmt.Printf(format, s.Lo, s.Hi, s.Stride)
} }
fmt.Print("\t},\n")
fmt.Print("}\n\n") fmt.Print("}\n\n")
} }
decl.Sort() decl.Sort()
@ -808,7 +834,7 @@ func printCaseRange(lo, hi *caseState) {
fmt.Printf("\t{0x%04X, 0x%04X, d{UpperLower, UpperLower, UpperLower}},\n", fmt.Printf("\t{0x%04X, 0x%04X, d{UpperLower, UpperLower, UpperLower}},\n",
lo.point, hi.point) lo.point, hi.point)
case hi.point > lo.point && lo.isLowerUpper(): case hi.point > lo.point && lo.isLowerUpper():
logger.Fatalf("LowerUpper sequence: should not happen: U+%04X. If it's real, need to fix To()", lo.point) logger.Fatalf("LowerUpper sequence: should not happen: %U. If it's real, need to fix To()", lo.point)
fmt.Printf("\t{0x%04X, 0x%04X, d{LowerUpper, LowerUpper, LowerUpper}},\n", fmt.Printf("\t{0x%04X, 0x%04X, d{LowerUpper, LowerUpper, LowerUpper}},\n",
lo.point, hi.point) lo.point, hi.point)
default: default:
@ -831,17 +857,28 @@ func fullCaseTest() {
lower := unicode.ToLower(i) lower := unicode.ToLower(i)
want := caseIt(i, c.lowerCase) want := caseIt(i, c.lowerCase)
if lower != want { if lower != want {
fmt.Fprintf(os.Stderr, "lower U+%04X should be U+%04X is U+%04X\n", i, want, lower) fmt.Fprintf(os.Stderr, "lower %U should be %U is %U\n", i, want, lower)
} }
upper := unicode.ToUpper(i) upper := unicode.ToUpper(i)
want = caseIt(i, c.upperCase) want = caseIt(i, c.upperCase)
if upper != want { if upper != want {
fmt.Fprintf(os.Stderr, "upper U+%04X should be U+%04X is U+%04X\n", i, want, upper) fmt.Fprintf(os.Stderr, "upper %U should be %U is %U\n", i, want, upper)
} }
title := unicode.ToTitle(i) title := unicode.ToTitle(i)
want = caseIt(i, c.titleCase) want = caseIt(i, c.titleCase)
if title != want { if title != want {
fmt.Fprintf(os.Stderr, "title U+%04X should be U+%04X is U+%04X\n", i, want, title) fmt.Fprintf(os.Stderr, "title %U should be %U is %U\n", i, want, title)
} }
} }
} }
var range16Count = 0 // Number of entries in the 16-bit range tables.
var range32Count = 0 // Number of entries in the 32-bit range tables.
func printSizes() {
fmt.Println()
fmt.Printf("// Range entries: %d 16-bit, %d 32-bit, %d total.\n", range16Count, range32Count, range16Count+range32Count)
range16Bytes := range16Count * 3 * 2
range32Bytes := range32Count * 3 * 4
fmt.Printf("// Range bytes: %d 16-bit, %d 32-bit, %d total.\n", range16Bytes, range32Bytes, range16Bytes+range32Bytes)
}

7639
src/pkg/unicode/tables.go
View File

File diff suppressed because it is too large Load Diff

612
src/pkg/xml/xml.go
View File

@ -1028,312 +1028,316 @@ func isNameByte(c byte) bool {
// and then reformatting. First corresponds to (Letter | '_' | ':') // and then reformatting. First corresponds to (Letter | '_' | ':')
// and second corresponds to NameChar. // and second corresponds to NameChar.
var first = []unicode.Range{ var first = &unicode.RangeTable{
{0x003A, 0x003A, 1}, R16: []unicode.Range16{
{0x0041, 0x005A, 1}, {0x003A, 0x003A, 1},
{0x005F, 0x005F, 1}, {0x0041, 0x005A, 1},
{0x0061, 0x007A, 1}, {0x005F, 0x005F, 1},
{0x00C0, 0x00D6, 1}, {0x0061, 0x007A, 1},
{0x00D8, 0x00F6, 1}, {0x00C0, 0x00D6, 1},
{0x00F8, 0x00FF, 1}, {0x00D8, 0x00F6, 1},
{0x0100, 0x0131, 1}, {0x00F8, 0x00FF, 1},
{0x0134, 0x013E, 1}, {0x0100, 0x0131, 1},
{0x0141, 0x0148, 1}, {0x0134, 0x013E, 1},
{0x014A, 0x017E, 1}, {0x0141, 0x0148, 1},
{0x0180, 0x01C3, 1}, {0x014A, 0x017E, 1},
{0x01CD, 0x01F0, 1}, {0x0180, 0x01C3, 1},
{0x01F4, 0x01F5, 1}, {0x01CD, 0x01F0, 1},
{0x01FA, 0x0217, 1}, {0x01F4, 0x01F5, 1},
{0x0250, 0x02A8, 1}, {0x01FA, 0x0217, 1},
{0x02BB, 0x02C1, 1}, {0x0250, 0x02A8, 1},
{0x0386, 0x0386, 1}, {0x02BB, 0x02C1, 1},
{0x0388, 0x038A, 1}, {0x0386, 0x0386, 1},
{0x038C, 0x038C, 1}, {0x0388, 0x038A, 1},
{0x038E, 0x03A1, 1}, {0x038C, 0x038C, 1},
{0x03A3, 0x03CE, 1}, {0x038E, 0x03A1, 1},
{0x03D0, 0x03D6, 1}, {0x03A3, 0x03CE, 1},
{0x03DA, 0x03E0, 2}, {0x03D0, 0x03D6, 1},
{0x03E2, 0x03F3, 1}, {0x03DA, 0x03E0, 2},
{0x0401, 0x040C, 1}, {0x03E2, 0x03F3, 1},
{0x040E, 0x044F, 1}, {0x0401, 0x040C, 1},
{0x0451, 0x045C, 1}, {0x040E, 0x044F, 1},
{0x045E, 0x0481, 1}, {0x0451, 0x045C, 1},
{0x0490, 0x04C4, 1}, {0x045E, 0x0481, 1},
{0x04C7, 0x04C8, 1}, {0x0490, 0x04C4, 1},
{0x04CB, 0x04CC, 1}, {0x04C7, 0x04C8, 1},
{0x04D0, 0x04EB, 1}, {0x04CB, 0x04CC, 1},
{0x04EE, 0x04F5, 1}, {0x04D0, 0x04EB, 1},
{0x04F8, 0x04F9, 1}, {0x04EE, 0x04F5, 1},
{0x0531, 0x0556, 1}, {0x04F8, 0x04F9, 1},
{0x0559, 0x0559, 1}, {0x0531, 0x0556, 1},
{0x0561, 0x0586, 1}, {0x0559, 0x0559, 1},
{0x05D0, 0x05EA, 1}, {0x0561, 0x0586, 1},
{0x05F0, 0x05F2, 1}, {0x05D0, 0x05EA, 1},
{0x0621, 0x063A, 1}, {0x05F0, 0x05F2, 1},
{0x0641, 0x064A, 1}, {0x0621, 0x063A, 1},
{0x0671, 0x06B7, 1}, {0x0641, 0x064A, 1},
{0x06BA, 0x06BE, 1}, {0x0671, 0x06B7, 1},
{0x06C0, 0x06CE, 1}, {0x06BA, 0x06BE, 1},
{0x06D0, 0x06D3, 1}, {0x06C0, 0x06CE, 1},
{0x06D5, 0x06D5, 1}, {0x06D0, 0x06D3, 1},
{0x06E5, 0x06E6, 1}, {0x06D5, 0x06D5, 1},
{0x0905, 0x0939, 1}, {0x06E5, 0x06E6, 1},
{0x093D, 0x093D, 1}, {0x0905, 0x0939, 1},
{0x0958, 0x0961, 1}, {0x093D, 0x093D, 1},
{0x0985, 0x098C, 1}, {0x0958, 0x0961, 1},
{0x098F, 0x0990, 1}, {0x0985, 0x098C, 1},
{0x0993, 0x09A8, 1}, {0x098F, 0x0990, 1},
{0x09AA, 0x09B0, 1}, {0x0993, 0x09A8, 1},
{0x09B2, 0x09B2, 1}, {0x09AA, 0x09B0, 1},
{0x09B6, 0x09B9, 1}, {0x09B2, 0x09B2, 1},
{0x09DC, 0x09DD, 1}, {0x09B6, 0x09B9, 1},
{0x09DF, 0x09E1, 1}, {0x09DC, 0x09DD, 1},
{0x09F0, 0x09F1, 1}, {0x09DF, 0x09E1, 1},
{0x0A05, 0x0A0A, 1}, {0x09F0, 0x09F1, 1},
{0x0A0F, 0x0A10, 1}, {0x0A05, 0x0A0A, 1},
{0x0A13, 0x0A28, 1}, {0x0A0F, 0x0A10, 1},
{0x0A2A, 0x0A30, 1}, {0x0A13, 0x0A28, 1},
{0x0A32, 0x0A33, 1}, {0x0A2A, 0x0A30, 1},
{0x0A35, 0x0A36, 1}, {0x0A32, 0x0A33, 1},
{0x0A38, 0x0A39, 1}, {0x0A35, 0x0A36, 1},
{0x0A59, 0x0A5C, 1}, {0x0A38, 0x0A39, 1},
{0x0A5E, 0x0A5E, 1}, {0x0A59, 0x0A5C, 1},
{0x0A72, 0x0A74, 1}, {0x0A5E, 0x0A5E, 1},
{0x0A85, 0x0A8B, 1}, {0x0A72, 0x0A74, 1},
{0x0A8D, 0x0A8D, 1}, {0x0A85, 0x0A8B, 1},
{0x0A8F, 0x0A91, 1}, {0x0A8D, 0x0A8D, 1},
{0x0A93, 0x0AA8, 1}, {0x0A8F, 0x0A91, 1},
{0x0AAA, 0x0AB0, 1}, {0x0A93, 0x0AA8, 1},
{0x0AB2, 0x0AB3, 1}, {0x0AAA, 0x0AB0, 1},
{0x0AB5, 0x0AB9, 1}, {0x0AB2, 0x0AB3, 1},
{0x0ABD, 0x0AE0, 0x23}, {0x0AB5, 0x0AB9, 1},
{0x0B05, 0x0B0C, 1}, {0x0ABD, 0x0AE0, 0x23},
{0x0B0F, 0x0B10, 1}, {0x0B05, 0x0B0C, 1},
{0x0B13, 0x0B28, 1}, {0x0B0F, 0x0B10, 1},
{0x0B2A, 0x0B30, 1}, {0x0B13, 0x0B28, 1},
{0x0B32, 0x0B33, 1}, {0x0B2A, 0x0B30, 1},
{0x0B36, 0x0B39, 1}, {0x0B32, 0x0B33, 1},
{0x0B3D, 0x0B3D, 1}, {0x0B36, 0x0B39, 1},
{0x0B5C, 0x0B5D, 1}, {0x0B3D, 0x0B3D, 1},
{0x0B5F, 0x0B61, 1}, {0x0B5C, 0x0B5D, 1},
{0x0B85, 0x0B8A, 1}, {0x0B5F, 0x0B61, 1},
{0x0B8E, 0x0B90, 1}, {0x0B85, 0x0B8A, 1},
{0x0B92, 0x0B95, 1}, {0x0B8E, 0x0B90, 1},
{0x0B99, 0x0B9A, 1}, {0x0B92, 0x0B95, 1},
{0x0B9C, 0x0B9C, 1}, {0x0B99, 0x0B9A, 1},
{0x0B9E, 0x0B9F, 1}, {0x0B9C, 0x0B9C, 1},
{0x0BA3, 0x0BA4, 1}, {0x0B9E, 0x0B9F, 1},
{0x0BA8, 0x0BAA, 1}, {0x0BA3, 0x0BA4, 1},
{0x0BAE, 0x0BB5, 1}, {0x0BA8, 0x0BAA, 1},
{0x0BB7, 0x0BB9, 1}, {0x0BAE, 0x0BB5, 1},
{0x0C05, 0x0C0C, 1}, {0x0BB7, 0x0BB9, 1},
{0x0C0E, 0x0C10, 1}, {0x0C05, 0x0C0C, 1},
{0x0C12, 0x0C28, 1}, {0x0C0E, 0x0C10, 1},
{0x0C2A, 0x0C33, 1}, {0x0C12, 0x0C28, 1},
{0x0C35, 0x0C39, 1}, {0x0C2A, 0x0C33, 1},
{0x0C60, 0x0C61, 1}, {0x0C35, 0x0C39, 1},
{0x0C85, 0x0C8C, 1}, {0x0C60, 0x0C61, 1},
{0x0C8E, 0x0C90, 1}, {0x0C85, 0x0C8C, 1},
{0x0C92, 0x0CA8, 1}, {0x0C8E, 0x0C90, 1},
{0x0CAA, 0x0CB3, 1}, {0x0C92, 0x0CA8, 1},
{0x0CB5, 0x0CB9, 1}, {0x0CAA, 0x0CB3, 1},
{0x0CDE, 0x0CDE, 1}, {0x0CB5, 0x0CB9, 1},
{0x0CE0, 0x0CE1, 1}, {0x0CDE, 0x0CDE, 1},
{0x0D05, 0x0D0C, 1}, {0x0CE0, 0x0CE1, 1},
{0x0D0E, 0x0D10, 1}, {0x0D05, 0x0D0C, 1},
{0x0D12, 0x0D28, 1}, {0x0D0E, 0x0D10, 1},
{0x0D2A, 0x0D39, 1}, {0x0D12, 0x0D28, 1},
{0x0D60, 0x0D61, 1}, {0x0D2A, 0x0D39, 1},
{0x0E01, 0x0E2E, 1}, {0x0D60, 0x0D61, 1},
{0x0E30, 0x0E30, 1}, {0x0E01, 0x0E2E, 1},
{0x0E32, 0x0E33, 1}, {0x0E30, 0x0E30, 1},
{0x0E40, 0x0E45, 1}, {0x0E32, 0x0E33, 1},
{0x0E81, 0x0E82, 1}, {0x0E40, 0x0E45, 1},
{0x0E84, 0x0E84, 1}, {0x0E81, 0x0E82, 1},
{0x0E87, 0x0E88, 1}, {0x0E84, 0x0E84, 1},
{0x0E8A, 0x0E8D, 3}, {0x0E87, 0x0E88, 1},
{0x0E94, 0x0E97, 1}, {0x0E8A, 0x0E8D, 3},
{0x0E99, 0x0E9F, 1}, {0x0E94, 0x0E97, 1},
{0x0EA1, 0x0EA3, 1}, {0x0E99, 0x0E9F, 1},
{0x0EA5, 0x0EA7, 2}, {0x0EA1, 0x0EA3, 1},
{0x0EAA, 0x0EAB, 1}, {0x0EA5, 0x0EA7, 2},
{0x0EAD, 0x0EAE, 1}, {0x0EAA, 0x0EAB, 1},
{0x0EB0, 0x0EB0, 1}, {0x0EAD, 0x0EAE, 1},
{0x0EB2, 0x0EB3, 1}, {0x0EB0, 0x0EB0, 1},
{0x0EBD, 0x0EBD, 1}, {0x0EB2, 0x0EB3, 1},
{0x0EC0, 0x0EC4, 1}, {0x0EBD, 0x0EBD, 1},
{0x0F40, 0x0F47, 1}, {0x0EC0, 0x0EC4, 1},
{0x0F49, 0x0F69, 1}, {0x0F40, 0x0F47, 1},
{0x10A0, 0x10C5, 1}, {0x0F49, 0x0F69, 1},
{0x10D0, 0x10F6, 1}, {0x10A0, 0x10C5, 1},
{0x1100, 0x1100, 1}, {0x10D0, 0x10F6, 1},
{0x1102, 0x1103, 1}, {0x1100, 0x1100, 1},
{0x1105, 0x1107, 1}, {0x1102, 0x1103, 1},
{0x1109, 0x1109, 1}, {0x1105, 0x1107, 1},
{0x110B, 0x110C, 1}, {0x1109, 0x1109, 1},
{0x110E, 0x1112, 1}, {0x110B, 0x110C, 1},
{0x113C, 0x1140, 2}, {0x110E, 0x1112, 1},
{0x114C, 0x1150, 2}, {0x113C, 0x1140, 2},
{0x1154, 0x1155, 1}, {0x114C, 0x1150, 2},
{0x1159, 0x1159, 1}, {0x1154, 0x1155, 1},
{0x115F, 0x1161, 1}, {0x1159, 0x1159, 1},
{0x1163, 0x1169, 2}, {0x115F, 0x1161, 1},
{0x116D, 0x116E, 1}, {0x1163, 0x1169, 2},
{0x1172, 0x1173, 1}, {0x116D, 0x116E, 1},
{0x1175, 0x119E, 0x119E - 0x1175}, {0x1172, 0x1173, 1},
{0x11A8, 0x11AB, 0x11AB - 0x11A8}, {0x1175, 0x119E, 0x119E - 0x1175},
{0x11AE, 0x11AF, 1}, {0x11A8, 0x11AB, 0x11AB - 0x11A8},
{0x11B7, 0x11B8, 1}, {0x11AE, 0x11AF, 1},
{0x11BA, 0x11BA, 1}, {0x11B7, 0x11B8, 1},
{0x11BC, 0x11C2, 1}, {0x11BA, 0x11BA, 1},
{0x11EB, 0x11F0, 0x11F0 - 0x11EB}, {0x11BC, 0x11C2, 1},
{0x11F9, 0x11F9, 1}, {0x11EB, 0x11F0, 0x11F0 - 0x11EB},
{0x1E00, 0x1E9B, 1}, {0x11F9, 0x11F9, 1},
{0x1EA0, 0x1EF9, 1}, {0x1E00, 0x1E9B, 1},
{0x1F00, 0x1F15, 1}, {0x1EA0, 0x1EF9, 1},
{0x1F18, 0x1F1D, 1}, {0x1F00, 0x1F15, 1},
{0x1F20, 0x1F45, 1}, {0x1F18, 0x1F1D, 1},
{0x1F48, 0x1F4D, 1}, {0x1F20, 0x1F45, 1},
{0x1F50, 0x1F57, 1}, {0x1F48, 0x1F4D, 1},
{0x1F59, 0x1F5B, 0x1F5B - 0x1F59}, {0x1F50, 0x1F57, 1},
{0x1F5D, 0x1F5D, 1}, {0x1F59, 0x1F5B, 0x1F5B - 0x1F59},
{0x1F5F, 0x1F7D, 1}, {0x1F5D, 0x1F5D, 1},
{0x1F80, 0x1FB4, 1}, {0x1F5F, 0x1F7D, 1},
{0x1FB6, 0x1FBC, 1}, {0x1F80, 0x1FB4, 1},
{0x1FBE, 0x1FBE, 1}, {0x1FB6, 0x1FBC, 1},
{0x1FC2, 0x1FC4, 1}, {0x1FBE, 0x1FBE, 1},
{0x1FC6, 0x1FCC, 1}, {0x1FC2, 0x1FC4, 1},
{0x1FD0, 0x1FD3, 1}, {0x1FC6, 0x1FCC, 1},
{0x1FD6, 0x1FDB, 1}, {0x1FD0, 0x1FD3, 1},
{0x1FE0, 0x1FEC, 1}, {0x1FD6, 0x1FDB, 1},
{0x1FF2, 0x1FF4, 1}, {0x1FE0, 0x1FEC, 1},
{0x1FF6, 0x1FFC, 1}, {0x1FF2, 0x1FF4, 1},
{0x2126, 0x2126, 1}, {0x1FF6, 0x1FFC, 1},
{0x212A, 0x212B, 1}, {0x2126, 0x2126, 1},
{0x212E, 0x212E, 1}, {0x212A, 0x212B, 1},
{0x2180, 0x2182, 1}, {0x212E, 0x212E, 1},
{0x3007, 0x3007, 1}, {0x2180, 0x2182, 1},
{0x3021, 0x3029, 1}, {0x3007, 0x3007, 1},
{0x3041, 0x3094, 1}, {0x3021, 0x3029, 1},
{0x30A1, 0x30FA, 1}, {0x3041, 0x3094, 1},
{0x3105, 0x312C, 1}, {0x30A1, 0x30FA, 1},
{0x4E00, 0x9FA5, 1}, {0x3105, 0x312C, 1},
{0xAC00, 0xD7A3, 1}, {0x4E00, 0x9FA5, 1},
{0xAC00, 0xD7A3, 1},
},
} }
var second = []unicode.Range{ var second = &unicode.RangeTable{
{0x002D, 0x002E, 1}, R16: []unicode.Range16{
{0x0030, 0x0039, 1}, {0x002D, 0x002E, 1},
{0x00B7, 0x00B7, 1}, {0x0030, 0x0039, 1},
{0x02D0, 0x02D1, 1}, {0x00B7, 0x00B7, 1},
{0x0300, 0x0345, 1}, {0x02D0, 0x02D1, 1},
{0x0360, 0x0361, 1}, {0x0300, 0x0345, 1},
{0x0387, 0x0387, 1}, {0x0360, 0x0361, 1},
{0x0483, 0x0486, 1}, {0x0387, 0x0387, 1},
{0x0591, 0x05A1, 1}, {0x0483, 0x0486, 1},
{0x05A3, 0x05B9, 1}, {0x0591, 0x05A1, 1},
{0x05BB, 0x05BD, 1}, {0x05A3, 0x05B9, 1},
{0x05BF, 0x05BF, 1}, {0x05BB, 0x05BD, 1},
{0x05C1, 0x05C2, 1}, {0x05BF, 0x05BF, 1},
{0x05C4, 0x0640, 0x0640 - 0x05C4}, {0x05C1, 0x05C2, 1},
{0x064B, 0x0652, 1}, {0x05C4, 0x0640, 0x0640 - 0x05C4},
{0x0660, 0x0669, 1}, {0x064B, 0x0652, 1},
{0x0670, 0x0670, 1}, {0x0660, 0x0669, 1},
{0x06D6, 0x06DC, 1}, {0x0670, 0x0670, 1},
{0x06DD, 0x06DF, 1}, {0x06D6, 0x06DC, 1},
{0x06E0, 0x06E4, 1}, {0x06DD, 0x06DF, 1},
{0x06E7, 0x06E8, 1}, {0x06E0, 0x06E4, 1},
{0x06EA, 0x06ED, 1}, {0x06E7, 0x06E8, 1},
{0x06F0, 0x06F9, 1}, {0x06EA, 0x06ED, 1},
{0x0901, 0x0903, 1}, {0x06F0, 0x06F9, 1},
{0x093C, 0x093C, 1}, {0x0901, 0x0903, 1},
{0x093E, 0x094C, 1}, {0x093C, 0x093C, 1},
{0x094D, 0x094D, 1}, {0x093E, 0x094C, 1},
{0x0951, 0x0954, 1}, {0x094D, 0x094D, 1},
{0x0962, 0x0963, 1}, {0x0951, 0x0954, 1},
{0x0966, 0x096F, 1}, {0x0962, 0x0963, 1},
{0x0981, 0x0983, 1}, {0x0966, 0x096F, 1},
{0x09BC, 0x09BC, 1}, {0x0981, 0x0983, 1},
{0x09BE, 0x09BF, 1}, {0x09BC, 0x09BC, 1},
{0x09C0, 0x09C4, 1}, {0x09BE, 0x09BF, 1},
{0x09C7, 0x09C8, 1}, {0x09C0, 0x09C4, 1},
{0x09CB, 0x09CD, 1}, {0x09C7, 0x09C8, 1},
{0x09D7, 0x09D7, 1}, {0x09CB, 0x09CD, 1},
{0x09E2, 0x09E3, 1}, {0x09D7, 0x09D7, 1},
{0x09E6, 0x09EF, 1}, {0x09E2, 0x09E3, 1},
{0x0A02, 0x0A3C, 0x3A}, {0x09E6, 0x09EF, 1},
{0x0A3E, 0x0A3F, 1}, {0x0A02, 0x0A3C, 0x3A},
{0x0A40, 0x0A42, 1}, {0x0A3E, 0x0A3F, 1},
{0x0A47, 0x0A48, 1}, {0x0A40, 0x0A42, 1},
{0x0A4B, 0x0A4D, 1}, {0x0A47, 0x0A48, 1},
{0x0A66, 0x0A6F, 1}, {0x0A4B, 0x0A4D, 1},
{0x0A70, 0x0A71, 1}, {0x0A66, 0x0A6F, 1},
{0x0A81, 0x0A83, 1}, {0x0A70, 0x0A71, 1},
{0x0ABC, 0x0ABC, 1}, {0x0A81, 0x0A83, 1},
{0x0ABE, 0x0AC5, 1}, {0x0ABC, 0x0ABC, 1},
{0x0AC7, 0x0AC9, 1}, {0x0ABE, 0x0AC5, 1},
{0x0ACB, 0x0ACD, 1}, {0x0AC7, 0x0AC9, 1},
{0x0AE6, 0x0AEF, 1}, {0x0ACB, 0x0ACD, 1},
{0x0B01, 0x0B03, 1}, {0x0AE6, 0x0AEF, 1},
{0x0B3C, 0x0B3C, 1}, {0x0B01, 0x0B03, 1},
{0x0B3E, 0x0B43, 1}, {0x0B3C, 0x0B3C, 1},
{0x0B47, 0x0B48, 1}, {0x0B3E, 0x0B43, 1},
{0x0B4B, 0x0B4D, 1}, {0x0B47, 0x0B48, 1},
{0x0B56, 0x0B57, 1}, {0x0B4B, 0x0B4D, 1},
{0x0B66, 0x0B6F, 1}, {0x0B56, 0x0B57, 1},
{0x0B82, 0x0B83, 1}, {0x0B66, 0x0B6F, 1},
{0x0BBE, 0x0BC2, 1}, {0x0B82, 0x0B83, 1},
{0x0BC6, 0x0BC8, 1}, {0x0BBE, 0x0BC2, 1},
{0x0BCA, 0x0BCD, 1}, {0x0BC6, 0x0BC8, 1},
{0x0BD7, 0x0BD7, 1}, {0x0BCA, 0x0BCD, 1},
{0x0BE7, 0x0BEF, 1}, {0x0BD7, 0x0BD7, 1},
{0x0C01, 0x0C03, 1}, {0x0BE7, 0x0BEF, 1},
{0x0C3E, 0x0C44, 1}, {0x0C01, 0x0C03, 1},
{0x0C46, 0x0C48, 1}, {0x0C3E, 0x0C44, 1},
{0x0C4A, 0x0C4D, 1}, {0x0C46, 0x0C48, 1},
{0x0C55, 0x0C56, 1}, {0x0C4A, 0x0C4D, 1},
{0x0C66, 0x0C6F, 1}, {0x0C55, 0x0C56, 1},
{0x0C82, 0x0C83, 1}, {0x0C66, 0x0C6F, 1},
{0x0CBE, 0x0CC4, 1}, {0x0C82, 0x0C83, 1},
{0x0CC6, 0x0CC8, 1}, {0x0CBE, 0x0CC4, 1},
{0x0CCA, 0x0CCD, 1}, {0x0CC6, 0x0CC8, 1},
{0x0CD5, 0x0CD6, 1}, {0x0CCA, 0x0CCD, 1},
{0x0CE6, 0x0CEF, 1}, {0x0CD5, 0x0CD6, 1},
{0x0D02, 0x0D03, 1}, {0x0CE6, 0x0CEF, 1},
{0x0D3E, 0x0D43, 1}, {0x0D02, 0x0D03, 1},
{0x0D46, 0x0D48, 1}, {0x0D3E, 0x0D43, 1},
{0x0D4A, 0x0D4D, 1}, {0x0D46, 0x0D48, 1},
{0x0D57, 0x0D57, 1}, {0x0D4A, 0x0D4D, 1},
{0x0D66, 0x0D6F, 1}, {0x0D57, 0x0D57, 1},
{0x0E31, 0x0E31, 1}, {0x0D66, 0x0D6F, 1},
{0x0E34, 0x0E3A, 1}, {0x0E31, 0x0E31, 1},
{0x0E46, 0x0E46, 1}, {0x0E34, 0x0E3A, 1},
{0x0E47, 0x0E4E, 1}, {0x0E46, 0x0E46, 1},
{0x0E50, 0x0E59, 1}, {0x0E47, 0x0E4E, 1},
{0x0EB1, 0x0EB1, 1}, {0x0E50, 0x0E59, 1},
{0x0EB4, 0x0EB9, 1}, {0x0EB1, 0x0EB1, 1},
{0x0EBB, 0x0EBC, 1}, {0x0EB4, 0x0EB9, 1},
{0x0EC6, 0x0EC6, 1}, {0x0EBB, 0x0EBC, 1},
{0x0EC8, 0x0ECD, 1}, {0x0EC6, 0x0EC6, 1},
{0x0ED0, 0x0ED9, 1}, {0x0EC8, 0x0ECD, 1},
{0x0F18, 0x0F19, 1}, {0x0ED0, 0x0ED9, 1},
{0x0F20, 0x0F29, 1}, {0x0F18, 0x0F19, 1},
{0x0F35, 0x0F39, 2}, {0x0F20, 0x0F29, 1},
{0x0F3E, 0x0F3F, 1}, {0x0F35, 0x0F39, 2},
{0x0F71, 0x0F84, 1}, {0x0F3E, 0x0F3F, 1},
{0x0F86, 0x0F8B, 1}, {0x0F71, 0x0F84, 1},
{0x0F90, 0x0F95, 1}, {0x0F86, 0x0F8B, 1},
{0x0F97, 0x0F97, 1}, {0x0F90, 0x0F95, 1},
{0x0F99, 0x0FAD, 1}, {0x0F97, 0x0F97, 1},
{0x0FB1, 0x0FB7, 1}, {0x0F99, 0x0FAD, 1},
{0x0FB9, 0x0FB9, 1}, {0x0FB1, 0x0FB7, 1},
{0x20D0, 0x20DC, 1}, {0x0FB9, 0x0FB9, 1},
{0x20E1, 0x3005, 0x3005 - 0x20E1}, {0x20D0, 0x20DC, 1},
{0x302A, 0x302F, 1}, {0x20E1, 0x3005, 0x3005 - 0x20E1},
{0x3031, 0x3035, 1}, {0x302A, 0x302F, 1},
{0x3099, 0x309A, 1}, {0x3031, 0x3035, 1},
{0x309D, 0x309E, 1}, {0x3099, 0x309A, 1},
{0x30FC, 0x30FE, 1}, {0x309D, 0x309E, 1},
{0x30FC, 0x30FE, 1},
},
} }
// HTMLEntity is an entity map containing translations for the // HTMLEntity is an entity map containing translations for the