2009-08-26 17:01:31 -06:00
|
|
|
// Copyright 2009 The Go Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
|
|
// Unicode table generator.
|
|
|
|
// Data read from the web.
|
|
|
|
|
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
2009-12-15 16:41:46 -07:00
|
|
|
"bufio"
|
|
|
|
"flag"
|
|
|
|
"fmt"
|
|
|
|
"http"
|
|
|
|
"log"
|
|
|
|
"os"
|
|
|
|
"sort"
|
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
"regexp"
|
|
|
|
"unicode"
|
2009-08-26 17:01:31 -06:00
|
|
|
)
|
|
|
|
|
2009-08-29 00:05:16 -06:00
|
|
|
func main() {
|
2009-12-15 16:41:46 -07:00
|
|
|
flag.Parse()
|
|
|
|
loadChars() // always needed
|
|
|
|
printCategories()
|
|
|
|
printScriptOrProperty(false)
|
|
|
|
printScriptOrProperty(true)
|
|
|
|
printCases()
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
|
2009-11-08 22:46:20 -07:00
|
|
|
var dataURL = flag.String("data", "", "full URL for UnicodeData.txt; defaults to --url/UnicodeData.txt")
|
2009-08-27 10:14:32 -06:00
|
|
|
var url = flag.String("url",
|
2011-01-31 16:20:44 -07:00
|
|
|
"http://www.unicode.org/Public/6.0.0/ucd/",
|
2009-08-27 18:04:23 -06:00
|
|
|
"URL of Unicode database directory")
|
|
|
|
var tablelist = flag.String("tables",
|
2009-08-27 10:14:32 -06:00
|
|
|
"all",
|
2009-11-02 12:37:52 -07:00
|
|
|
"comma-separated list of which tables to generate; can be letter")
|
2009-08-27 18:04:23 -06:00
|
|
|
var scriptlist = flag.String("scripts",
|
|
|
|
"all",
|
2009-11-02 12:37:52 -07:00
|
|
|
"comma-separated list of which script tables to generate")
|
2009-08-31 17:43:17 -06:00
|
|
|
var proplist = flag.String("props",
|
|
|
|
"all",
|
2009-11-02 12:37:52 -07:00
|
|
|
"comma-separated list of which property tables to generate")
|
2009-08-29 00:05:16 -06:00
|
|
|
var cases = flag.Bool("cases",
|
|
|
|
true,
|
2009-11-02 12:37:52 -07:00
|
|
|
"generate case tables")
|
2009-08-27 10:14:32 -06:00
|
|
|
var test = flag.Bool("test",
|
|
|
|
false,
|
2009-11-02 12:37:52 -07:00
|
|
|
"test existing tables; can be used to compare web data with package data")
|
2009-08-26 17:01:31 -06:00
|
|
|
|
2011-01-30 14:09:16 -07:00
|
|
|
var scriptRe = regexp.MustCompile(`^([0-9A-F]+)(\.\.[0-9A-F]+)? *; ([A-Za-z_]+)$`)
|
|
|
|
var logger = log.New(os.Stderr, "", log.Lshortfile)
|
2009-08-26 17:01:31 -06:00
|
|
|
|
2009-12-15 16:41:46 -07:00
|
|
|
var category = map[string]bool{"letter": true} // Nd Lu etc. letter is a special case
|
2009-08-27 10:14:32 -06:00
|
|
|
|
2009-08-27 18:04:23 -06:00
|
|
|
// UnicodeData.txt has form:
|
2009-08-26 17:01:31 -06:00
|
|
|
// 0037;DIGIT SEVEN;Nd;0;EN;;7;7;7;N;;;;;
|
|
|
|
// 007A;LATIN SMALL LETTER Z;Ll;0;L;;;;;N;;;005A;;005A
|
|
|
|
// See http://www.unicode.org/Public/5.1.0/ucd/UCD.html for full explanation
|
2009-08-29 00:05:16 -06:00
|
|
|
// The fields:
|
2009-08-26 17:01:31 -06:00
|
|
|
const (
|
2009-12-15 16:41:46 -07:00
|
|
|
FCodePoint = iota
|
|
|
|
FName
|
|
|
|
FGeneralCategory
|
|
|
|
FCanonicalCombiningClass
|
|
|
|
FBidiClass
|
|
|
|
FDecompositionType
|
|
|
|
FDecompositionMapping
|
|
|
|
FNumericType
|
|
|
|
FNumericValue
|
|
|
|
FBidiMirrored
|
|
|
|
FUnicode1Name
|
|
|
|
FISOComment
|
|
|
|
FSimpleUppercaseMapping
|
|
|
|
FSimpleLowercaseMapping
|
|
|
|
FSimpleTitlecaseMapping
|
|
|
|
NumField
|
|
|
|
|
|
|
|
MaxChar = 0x10FFFF // anything above this shouldn't exist
|
2009-08-26 17:01:31 -06:00
|
|
|
)
|
|
|
|
|
|
|
|
var fieldName = []string{
|
|
|
|
"CodePoint",
|
|
|
|
"Name",
|
|
|
|
"GeneralCategory",
|
|
|
|
"CanonicalCombiningClass",
|
|
|
|
"BidiClass",
|
|
|
|
"DecompositionType",
|
|
|
|
"DecompositionMapping",
|
|
|
|
"NumericType",
|
|
|
|
"NumericValue",
|
|
|
|
"BidiMirrored",
|
|
|
|
"Unicode1Name",
|
|
|
|
"ISOComment",
|
|
|
|
"SimpleUppercaseMapping",
|
|
|
|
"SimpleLowercaseMapping",
|
2009-11-04 22:39:55 -07:00
|
|
|
"SimpleTitlecaseMapping",
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
// This contains only the properties we're interested in.
|
|
|
|
type Char struct {
|
2009-12-15 16:41:46 -07:00
|
|
|
field []string // debugging only; could be deleted if we take out char.dump()
|
|
|
|
codePoint uint32 // if zero, this index is not a valid code point.
|
|
|
|
category string
|
|
|
|
upperCase int
|
|
|
|
lowerCase int
|
|
|
|
titleCase int
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
|
2009-08-27 18:04:23 -06:00
|
|
|
// Scripts.txt has form:
|
|
|
|
// A673 ; Cyrillic # Po SLAVONIC ASTERISK
|
|
|
|
// A67C..A67D ; Cyrillic # Mn [2] COMBINING CYRILLIC KAVYKA..COMBINING CYRILLIC PAYEROK
|
|
|
|
// See http://www.unicode.org/Public/5.1.0/ucd/UCD.html for full explanation
|
|
|
|
|
|
|
|
type Script struct {
|
2009-12-15 16:41:46 -07:00
|
|
|
lo, hi uint32 // range of code points
|
|
|
|
script string
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
|
|
|
|
2009-08-29 00:05:16 -06:00
|
|
|
var chars = make([]Char, MaxChar+1)
|
2009-11-04 22:39:55 -07:00
|
|
|
var scripts = make(map[string][]Script)
|
2009-12-15 16:41:46 -07:00
|
|
|
var props = make(map[string][]Script) // a property looks like a script; can share the format
|
2009-08-31 17:43:17 -06:00
|
|
|
|
|
|
|
var lastChar uint32 = 0
|
2009-08-26 17:01:31 -06:00
|
|
|
|
2009-08-28 12:57:38 -06:00
|
|
|
// In UnicodeData.txt, some ranges are marked like this:
|
2009-08-29 00:05:16 -06:00
|
|
|
// 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
|
|
|
|
// 4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
|
2009-08-28 12:57:38 -06:00
|
|
|
// parseCategory returns a state variable indicating the weirdness.
|
|
|
|
type State int
|
2009-11-04 22:39:55 -07:00
|
|
|
|
2009-08-28 12:57:38 -06:00
|
|
|
const (
|
2009-12-15 16:41:46 -07:00
|
|
|
SNormal State = iota // known to be zero for the type
|
|
|
|
SFirst
|
|
|
|
SLast
|
|
|
|
SMissing
|
2009-08-28 12:57:38 -06:00
|
|
|
)
|
|
|
|
|
|
|
|
func parseCategory(line string) (state State) {
|
2009-12-15 16:41:46 -07:00
|
|
|
field := strings.Split(line, ";", -1)
|
2009-08-26 17:01:31 -06:00
|
|
|
if len(field) != NumField {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatalf("%5s: %d fields (expected %d)\n", line, len(field), NumField)
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
point, err := strconv.Btoui64(field[FCodePoint], 16)
|
2009-08-26 17:01:31 -06:00
|
|
|
if err != nil {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatalf("%.5s...: %s", line, err)
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
lastChar = uint32(point)
|
2009-08-26 17:01:31 -06:00
|
|
|
if point == 0 {
|
2009-12-15 16:41:46 -07:00
|
|
|
return // not interesting and we use 0 as unset
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-08-29 00:05:16 -06:00
|
|
|
if point > MaxChar {
|
2009-11-09 13:07:39 -07:00
|
|
|
return
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
char := &chars[point]
|
|
|
|
char.field = field
|
2009-08-26 17:01:31 -06:00
|
|
|
if char.codePoint != 0 {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatalf("point %U reused", point)
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
char.codePoint = lastChar
|
|
|
|
char.category = field[FGeneralCategory]
|
|
|
|
category[char.category] = true
|
2009-08-26 17:01:31 -06:00
|
|
|
switch char.category {
|
|
|
|
case "Nd":
|
|
|
|
// Decimal digit
|
2009-12-15 16:41:46 -07:00
|
|
|
_, err := strconv.Atoi(field[FNumericValue])
|
2009-08-26 17:01:31 -06:00
|
|
|
if err != nil {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatalf("%U: bad numeric field: %s", point, err)
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
case "Lu":
|
2009-11-09 13:07:39 -07:00
|
|
|
char.letter(field[FCodePoint], field[FSimpleLowercaseMapping], field[FSimpleTitlecaseMapping])
|
2009-08-26 17:01:31 -06:00
|
|
|
case "Ll":
|
2009-11-09 13:07:39 -07:00
|
|
|
char.letter(field[FSimpleUppercaseMapping], field[FCodePoint], field[FSimpleTitlecaseMapping])
|
2009-08-26 17:01:31 -06:00
|
|
|
case "Lt":
|
2009-11-09 13:07:39 -07:00
|
|
|
char.letter(field[FSimpleUppercaseMapping], field[FSimpleLowercaseMapping], field[FCodePoint])
|
2009-08-26 17:01:31 -06:00
|
|
|
case "Lm", "Lo":
|
2009-11-09 13:07:39 -07:00
|
|
|
char.letter(field[FSimpleUppercaseMapping], field[FSimpleLowercaseMapping], field[FSimpleTitlecaseMapping])
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-08-28 12:57:38 -06:00
|
|
|
switch {
|
|
|
|
case strings.Index(field[FName], ", First>") > 0:
|
2009-11-09 13:07:39 -07:00
|
|
|
state = SFirst
|
2009-08-28 12:57:38 -06:00
|
|
|
case strings.Index(field[FName], ", Last>") > 0:
|
2009-11-09 13:07:39 -07:00
|
|
|
state = SLast
|
2009-08-28 12:57:38 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
return
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
func (char *Char) dump(s string) {
|
2009-12-15 16:41:46 -07:00
|
|
|
fmt.Print(s, " ")
|
2009-11-04 22:39:55 -07:00
|
|
|
for i := 0; i < len(char.field); i++ {
|
2009-11-09 13:07:39 -07:00
|
|
|
fmt.Printf("%s:%q ", fieldName[i], char.field[i])
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
fmt.Print("\n")
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
func (char *Char) letter(u, l, t string) {
|
2009-12-15 16:41:46 -07:00
|
|
|
char.upperCase = char.letterValue(u, "U")
|
|
|
|
char.lowerCase = char.letterValue(l, "L")
|
|
|
|
char.titleCase = char.letterValue(t, "T")
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
|
2009-08-29 00:05:16 -06:00
|
|
|
func (char *Char) letterValue(s string, cas string) int {
|
2009-08-26 17:01:31 -06:00
|
|
|
if s == "" {
|
2009-11-09 13:07:39 -07:00
|
|
|
return 0
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
v, err := strconv.Btoui64(s, 16)
|
2009-08-26 17:01:31 -06:00
|
|
|
if err != nil {
|
2009-12-15 16:41:46 -07:00
|
|
|
char.dump(cas)
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatalf("%U: bad letter(%s): %s", char.codePoint, s, err)
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
return int(v)
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
|
2009-08-27 10:14:32 -06:00
|
|
|
func allCategories() []string {
|
2009-12-15 16:41:46 -07:00
|
|
|
a := make([]string, len(category))
|
|
|
|
i := 0
|
2009-08-27 10:14:32 -06:00
|
|
|
for k := range category {
|
2009-12-15 16:41:46 -07:00
|
|
|
a[i] = k
|
|
|
|
i++
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
return a
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
|
|
|
|
2009-11-04 22:39:55 -07:00
|
|
|
func all(scripts map[string][]Script) []string {
|
2009-12-15 16:41:46 -07:00
|
|
|
a := make([]string, len(scripts))
|
|
|
|
i := 0
|
2009-08-27 18:04:23 -06:00
|
|
|
for k := range scripts {
|
2009-12-15 16:41:46 -07:00
|
|
|
a[i] = k
|
|
|
|
i++
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
return a
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
|
|
|
|
2009-08-27 10:14:32 -06:00
|
|
|
// Extract the version number from the URL
|
|
|
|
func version() string {
|
|
|
|
// Break on slashes and look for the first numeric field
|
2010-07-01 15:08:14 -06:00
|
|
|
fields := strings.Split(*url, "/", -1)
|
2009-08-27 10:14:32 -06:00
|
|
|
for _, f := range fields {
|
|
|
|
if len(f) > 0 && '0' <= f[0] && f[0] <= '9' {
|
2009-11-09 13:07:39 -07:00
|
|
|
return f
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
|
|
|
}
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatal("unknown version")
|
2009-12-15 16:41:46 -07:00
|
|
|
return "Unknown"
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
func letterOp(code int) bool {
|
|
|
|
switch chars[code].category {
|
|
|
|
case "Lu", "Ll", "Lt", "Lm", "Lo":
|
2009-11-09 13:07:39 -07:00
|
|
|
return true
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
return false
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
|
|
|
|
2009-08-29 00:05:16 -06:00
|
|
|
func loadChars() {
|
2009-11-08 22:46:20 -07:00
|
|
|
if *dataURL == "" {
|
2009-11-09 22:23:52 -07:00
|
|
|
flag.Set("data", *url+"UnicodeData.txt")
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
resp, _, err := http.Get(*dataURL)
|
2009-08-26 17:01:31 -06:00
|
|
|
if err != nil {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatal(err)
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-08-27 10:14:32 -06:00
|
|
|
if resp.StatusCode != 200 {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatal("bad GET status for UnicodeData.txt", resp.Status)
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
input := bufio.NewReader(resp.Body)
|
|
|
|
var first uint32 = 0
|
2009-08-26 17:01:31 -06:00
|
|
|
for {
|
2009-12-15 16:41:46 -07:00
|
|
|
line, err := input.ReadString('\n')
|
2009-08-26 17:01:31 -06:00
|
|
|
if err != nil {
|
|
|
|
if err == os.EOF {
|
2009-11-09 13:07:39 -07:00
|
|
|
break
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatal(err)
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-11-04 22:39:55 -07:00
|
|
|
switch parseCategory(line[0 : len(line)-1]) {
|
2009-08-28 12:57:38 -06:00
|
|
|
case SNormal:
|
|
|
|
if first != 0 {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatalf("bad state normal at U+%04X", lastChar)
|
2009-08-28 12:57:38 -06:00
|
|
|
}
|
|
|
|
case SFirst:
|
|
|
|
if first != 0 {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatalf("bad state first at U+%04X", lastChar)
|
2009-08-28 12:57:38 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
first = lastChar
|
2009-08-28 12:57:38 -06:00
|
|
|
case SLast:
|
|
|
|
if first == 0 {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatalf("bad state last at U+%04X", lastChar)
|
2009-08-28 12:57:38 -06:00
|
|
|
}
|
2009-11-09 22:23:52 -07:00
|
|
|
for i := first + 1; i <= lastChar; i++ {
|
2009-12-15 16:41:46 -07:00
|
|
|
chars[i] = chars[first]
|
|
|
|
chars[i].codePoint = i
|
2009-08-28 12:57:38 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
first = 0
|
2009-08-28 12:57:38 -06:00
|
|
|
}
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
resp.Body.Close()
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
func printCategories() {
|
|
|
|
if *tablelist == "" {
|
2009-11-09 13:07:39 -07:00
|
|
|
return
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
2009-08-27 10:14:32 -06:00
|
|
|
// Find out which categories to dump
|
2010-07-01 15:08:14 -06:00
|
|
|
list := strings.Split(*tablelist, ",", -1)
|
2009-08-27 18:04:23 -06:00
|
|
|
if *tablelist == "all" {
|
2009-11-09 13:07:39 -07:00
|
|
|
list = allCategories()
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
|
|
|
if *test {
|
2009-12-15 16:41:46 -07:00
|
|
|
fullCategoryTest(list)
|
|
|
|
return
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
2009-08-26 17:01:31 -06:00
|
|
|
fmt.Printf(
|
2009-12-09 17:54:07 -07:00
|
|
|
"// Generated by running\n"+
|
|
|
|
"// maketables --tables=%s --data=%s\n"+
|
|
|
|
"// DO NOT EDIT\n\n"+
|
2009-11-05 19:26:16 -07:00
|
|
|
"package unicode\n\n",
|
2009-08-27 18:04:23 -06:00
|
|
|
*tablelist,
|
2009-12-15 16:41:46 -07:00
|
|
|
*dataURL)
|
2009-08-27 10:14:32 -06:00
|
|
|
|
2009-12-15 16:41:46 -07:00
|
|
|
fmt.Println("// Version is the Unicode edition from which the tables are derived.")
|
|
|
|
fmt.Printf("const Version = %q\n\n", version())
|
2009-08-27 10:14:32 -06:00
|
|
|
|
2009-08-27 18:04:23 -06:00
|
|
|
if *tablelist == "all" {
|
2009-12-15 16:41:46 -07:00
|
|
|
fmt.Println("// Categories is the set of Unicode data tables.")
|
|
|
|
fmt.Println("var Categories = map[string] []Range {")
|
2010-12-08 22:36:56 -07:00
|
|
|
for k := range category {
|
2009-11-09 13:07:39 -07:00
|
|
|
fmt.Printf("\t%q: %s,\n", k, k)
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
2010-12-07 14:42:54 -07:00
|
|
|
fmt.Print("}\n\n")
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
|
|
|
|
2009-12-15 16:41:46 -07:00
|
|
|
decl := make(sort.StringArray, len(list))
|
|
|
|
ndecl := 0
|
2009-08-27 10:14:32 -06:00
|
|
|
for _, name := range list {
|
|
|
|
if _, ok := category[name]; !ok {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatal("unknown category", name)
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
|
|
|
// We generate an UpperCase name to serve as concise documentation and an _UnderScored
|
|
|
|
// name to store the data. This stops godoc dumping all the tables but keeps them
|
|
|
|
// available to clients.
|
|
|
|
// Cases deserving special comments
|
2009-12-15 16:41:46 -07:00
|
|
|
varDecl := ""
|
2009-08-27 10:14:32 -06:00
|
|
|
switch name {
|
2009-08-27 19:38:02 -06:00
|
|
|
case "letter":
|
2009-11-09 13:07:39 -07:00
|
|
|
varDecl = "\tLetter = letter; // Letter is the set of Unicode letters.\n"
|
2009-08-27 10:14:32 -06:00
|
|
|
case "Nd":
|
2009-11-09 13:07:39 -07:00
|
|
|
varDecl = "\tDigit = _Nd; // Digit is the set of Unicode characters with the \"decimal digit\" property.\n"
|
2009-08-27 10:14:32 -06:00
|
|
|
case "Lu":
|
2009-11-09 13:07:39 -07:00
|
|
|
varDecl = "\tUpper = _Lu; // Upper is the set of Unicode upper case letters.\n"
|
2009-08-27 10:14:32 -06:00
|
|
|
case "Ll":
|
2009-11-09 13:07:39 -07:00
|
|
|
varDecl = "\tLower = _Ll; // Lower is the set of Unicode lower case letters.\n"
|
2009-08-27 10:14:32 -06:00
|
|
|
case "Lt":
|
2009-11-09 13:07:39 -07:00
|
|
|
varDecl = "\tTitle = _Lt; // Title is the set of Unicode title case letters.\n"
|
2009-08-27 19:38:02 -06:00
|
|
|
}
|
|
|
|
if name != "letter" {
|
|
|
|
varDecl += fmt.Sprintf(
|
|
|
|
"\t%s = _%s; // %s is the set of Unicode characters in category %s.\n",
|
2009-11-09 13:07:39 -07:00
|
|
|
name, name, name, name)
|
2009-08-27 19:38:02 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
decl[ndecl] = varDecl
|
|
|
|
ndecl++
|
|
|
|
if name == "letter" { // special case
|
2009-08-27 19:38:02 -06:00
|
|
|
dumpRange(
|
|
|
|
"var letter = []Range {\n",
|
2009-12-15 16:41:46 -07:00
|
|
|
letterOp)
|
|
|
|
continue
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
2009-08-26 17:01:31 -06:00
|
|
|
dumpRange(
|
2009-08-27 19:38:02 -06:00
|
|
|
fmt.Sprintf("var _%s = []Range {\n", name),
|
2009-12-15 16:41:46 -07:00
|
|
|
func(code int) bool { return chars[code].category == name })
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
decl.Sort()
|
|
|
|
fmt.Println("var (")
|
2009-08-27 19:38:02 -06:00
|
|
|
for _, d := range decl {
|
2009-11-09 13:07:39 -07:00
|
|
|
fmt.Print(d)
|
2009-08-27 19:38:02 -06:00
|
|
|
}
|
2010-12-07 14:42:54 -07:00
|
|
|
fmt.Print(")\n\n")
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
type Op func(code int) bool
|
2009-11-04 22:39:55 -07:00
|
|
|
|
2011-01-31 16:20:44 -07:00
|
|
|
const format = "\t{0x%04x, 0x%04x, %d},\n"
|
2009-08-26 17:01:31 -06:00
|
|
|
|
2009-08-27 19:38:02 -06:00
|
|
|
func dumpRange(header string, inCategory Op) {
|
2009-12-15 16:41:46 -07:00
|
|
|
fmt.Print(header)
|
|
|
|
next := 0
|
2009-08-26 17:01:31 -06:00
|
|
|
// one Range for each iteration
|
|
|
|
for {
|
|
|
|
// look for start of range
|
|
|
|
for next < len(chars) && !inCategory(next) {
|
2009-11-09 13:07:39 -07:00
|
|
|
next++
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
if next >= len(chars) {
|
|
|
|
// no characters remain
|
2009-11-09 13:07:39 -07:00
|
|
|
break
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
// start of range
|
2009-12-15 16:41:46 -07:00
|
|
|
lo := next
|
|
|
|
hi := next
|
|
|
|
stride := 1
|
2009-08-26 17:01:31 -06:00
|
|
|
// accept lo
|
2009-12-15 16:41:46 -07:00
|
|
|
next++
|
2009-08-26 17:01:31 -06:00
|
|
|
// look for another character to set the stride
|
|
|
|
for next < len(chars) && !inCategory(next) {
|
2009-11-09 13:07:39 -07:00
|
|
|
next++
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
if next >= len(chars) {
|
|
|
|
// no more characters
|
2009-12-15 16:41:46 -07:00
|
|
|
fmt.Printf(format, lo, hi, stride)
|
|
|
|
break
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
// set stride
|
2009-12-15 16:41:46 -07:00
|
|
|
stride = next - lo
|
2009-08-26 17:01:31 -06:00
|
|
|
// check for length of run. next points to first jump in stride
|
|
|
|
for i := next; i < len(chars); i++ {
|
2009-11-09 22:23:52 -07:00
|
|
|
if inCategory(i) == (((i - lo) % stride) == 0) {
|
2009-08-26 17:01:31 -06:00
|
|
|
// accept
|
|
|
|
if inCategory(i) {
|
2009-11-09 13:07:39 -07:00
|
|
|
hi = i
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// no more characters in this run
|
2009-11-09 13:07:39 -07:00
|
|
|
break
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
fmt.Printf(format, lo, hi, stride)
|
2009-08-26 17:01:31 -06:00
|
|
|
// next range: start looking where this range ends
|
2009-12-15 16:41:46 -07:00
|
|
|
next = hi + 1
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
fmt.Print("}\n\n")
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-08-27 10:14:32 -06:00
|
|
|
|
2009-08-27 18:04:23 -06:00
|
|
|
func fullCategoryTest(list []string) {
|
2009-08-27 10:14:32 -06:00
|
|
|
for _, name := range list {
|
|
|
|
if _, ok := category[name]; !ok {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatal("unknown category", name)
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
r, ok := unicode.Categories[name]
|
2009-08-27 10:14:32 -06:00
|
|
|
if !ok {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatal("unknown table", name)
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
|
|
|
if name == "letter" {
|
2009-11-09 13:07:39 -07:00
|
|
|
verifyRange(name, letterOp, r)
|
2009-08-27 10:14:32 -06:00
|
|
|
} else {
|
|
|
|
verifyRange(
|
|
|
|
name,
|
|
|
|
func(code int) bool { return chars[code].category == name },
|
2009-11-09 13:07:39 -07:00
|
|
|
r)
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func verifyRange(name string, inCategory Op, table []unicode.Range) {
|
2009-11-02 12:37:52 -07:00
|
|
|
for i := range chars {
|
2009-12-15 16:41:46 -07:00
|
|
|
web := inCategory(i)
|
|
|
|
pkg := unicode.Is(table, i)
|
2009-08-27 10:14:32 -06:00
|
|
|
if web != pkg {
|
2009-11-09 13:07:39 -07:00
|
|
|
fmt.Fprintf(os.Stderr, "%s: U+%04X: web=%t pkg=%t\n", name, i, web, pkg)
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2009-08-27 18:04:23 -06:00
|
|
|
|
2009-11-04 22:39:55 -07:00
|
|
|
func parseScript(line string, scripts map[string][]Script) {
|
2009-12-15 16:41:46 -07:00
|
|
|
comment := strings.Index(line, "#")
|
2009-08-27 18:04:23 -06:00
|
|
|
if comment >= 0 {
|
2009-11-09 13:07:39 -07:00
|
|
|
line = line[0:comment]
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
line = strings.TrimSpace(line)
|
2009-08-27 18:04:23 -06:00
|
|
|
if len(line) == 0 {
|
2009-11-09 13:07:39 -07:00
|
|
|
return
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
field := strings.Split(line, ";", -1)
|
2009-08-27 18:04:23 -06:00
|
|
|
if len(field) != 2 {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatalf("%s: %d fields (expected 2)\n", line, len(field))
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
2011-01-30 14:09:16 -07:00
|
|
|
matches := scriptRe.FindStringSubmatch(line)
|
2009-08-27 18:04:23 -06:00
|
|
|
if len(matches) != 4 {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatalf("%s: %d matches (expected 3)\n", line, len(matches))
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
lo, err := strconv.Btoui64(matches[1], 16)
|
2009-08-27 18:04:23 -06:00
|
|
|
if err != nil {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatalf("%.5s...: %s", line, err)
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
hi := lo
|
|
|
|
if len(matches[2]) > 2 { // ignore leading ..
|
|
|
|
hi, err = strconv.Btoui64(matches[2][2:], 16)
|
2009-08-27 18:04:23 -06:00
|
|
|
if err != nil {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatalf("%.5s...: %s", line, err)
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
name := matches[3]
|
2010-10-27 20:47:23 -06:00
|
|
|
scripts[name] = append(scripts[name], Script{uint32(lo), uint32(hi), name})
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
|
|
|
|
2009-08-31 17:43:17 -06:00
|
|
|
// The script tables have a lot of adjacent elements. Fold them together.
|
|
|
|
func foldAdjacent(r []Script) []unicode.Range {
|
2009-12-15 16:41:46 -07:00
|
|
|
s := make([]unicode.Range, 0, len(r))
|
|
|
|
j := 0
|
2009-08-31 17:43:17 -06:00
|
|
|
for i := 0; i < len(r); i++ {
|
2009-11-09 22:23:52 -07:00
|
|
|
if j > 0 && int(r[i].lo) == s[j-1].Hi+1 {
|
2009-11-09 13:07:39 -07:00
|
|
|
s[j-1].Hi = int(r[i].hi)
|
2009-08-31 17:43:17 -06:00
|
|
|
} else {
|
2009-12-15 16:41:46 -07:00
|
|
|
s = s[0 : j+1]
|
|
|
|
s[j] = unicode.Range{int(r[i].lo), int(r[i].hi), 1}
|
|
|
|
j++
|
2009-08-31 17:43:17 -06:00
|
|
|
}
|
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
return s
|
2009-08-31 17:43:17 -06:00
|
|
|
}
|
|
|
|
|
2009-11-04 22:39:55 -07:00
|
|
|
func fullScriptTest(list []string, installed map[string][]unicode.Range, scripts map[string][]Script) {
|
2009-08-31 17:43:17 -06:00
|
|
|
for _, name := range list {
|
|
|
|
if _, ok := scripts[name]; !ok {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatal("unknown script", name)
|
2009-08-31 17:43:17 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
_, ok := installed[name]
|
2009-08-31 17:43:17 -06:00
|
|
|
if !ok {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatal("unknown table", name)
|
2009-08-31 17:43:17 -06:00
|
|
|
}
|
|
|
|
for _, script := range scripts[name] {
|
|
|
|
for r := script.lo; r <= script.hi; r++ {
|
|
|
|
if !unicode.Is(installed[name], int(r)) {
|
2009-11-09 13:07:39 -07:00
|
|
|
fmt.Fprintf(os.Stderr, "U+%04X: not in script %s\n", r, name)
|
2009-08-31 17:43:17 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// PropList.txt has the same format as Scripts.txt so we can share its parser.
|
|
|
|
func printScriptOrProperty(doProps bool) {
|
2009-12-15 16:41:46 -07:00
|
|
|
flag := "scripts"
|
|
|
|
flaglist := *scriptlist
|
|
|
|
file := "Scripts.txt"
|
|
|
|
table := scripts
|
|
|
|
installed := unicode.Scripts
|
2009-08-31 17:43:17 -06:00
|
|
|
if doProps {
|
2009-12-15 16:41:46 -07:00
|
|
|
flag = "props"
|
|
|
|
flaglist = *proplist
|
|
|
|
file = "PropList.txt"
|
|
|
|
table = props
|
|
|
|
installed = unicode.Properties
|
2009-08-31 17:43:17 -06:00
|
|
|
}
|
|
|
|
if flaglist == "" {
|
2009-11-09 13:07:39 -07:00
|
|
|
return
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
var err os.Error
|
|
|
|
resp, _, err := http.Get(*url + file)
|
2009-08-27 18:04:23 -06:00
|
|
|
if err != nil {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatal(err)
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
|
|
|
if resp.StatusCode != 200 {
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatal("bad GET status for ", file, ":", resp.Status)
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
input := bufio.NewReader(resp.Body)
|
2009-08-27 18:04:23 -06:00
|
|
|
for {
|
2009-12-15 16:41:46 -07:00
|
|
|
line, err := input.ReadString('\n')
|
2009-08-27 18:04:23 -06:00
|
|
|
if err != nil {
|
|
|
|
if err == os.EOF {
|
2009-11-09 13:07:39 -07:00
|
|
|
break
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatal(err)
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
parseScript(line[0:len(line)-1], table)
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
resp.Body.Close()
|
2009-08-27 18:04:23 -06:00
|
|
|
|
|
|
|
// Find out which scripts to dump
|
2010-07-01 15:08:14 -06:00
|
|
|
list := strings.Split(flaglist, ",", -1)
|
2009-08-31 17:43:17 -06:00
|
|
|
if flaglist == "all" {
|
2009-11-09 13:07:39 -07:00
|
|
|
list = all(table)
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
|
|
|
if *test {
|
2009-12-15 16:41:46 -07:00
|
|
|
fullScriptTest(list, installed, table)
|
|
|
|
return
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
fmt.Printf(
|
2009-12-09 17:54:07 -07:00
|
|
|
"// Generated by running\n"+
|
|
|
|
"// maketables --%s=%s --url=%s\n"+
|
2009-11-05 19:26:16 -07:00
|
|
|
"// DO NOT EDIT\n\n",
|
2009-08-31 17:43:17 -06:00
|
|
|
flag,
|
|
|
|
flaglist,
|
2009-12-15 16:41:46 -07:00
|
|
|
*url)
|
2009-08-31 17:43:17 -06:00
|
|
|
if flaglist == "all" {
|
|
|
|
if doProps {
|
2009-12-15 16:41:46 -07:00
|
|
|
fmt.Println("// Properties is the set of Unicode property tables.")
|
|
|
|
fmt.Println("var Properties = map[string] []Range {")
|
2009-08-31 17:43:17 -06:00
|
|
|
} else {
|
2009-12-15 16:41:46 -07:00
|
|
|
fmt.Println("// Scripts is the set of Unicode script tables.")
|
|
|
|
fmt.Println("var Scripts = map[string] []Range {")
|
2009-08-31 17:43:17 -06:00
|
|
|
}
|
2010-12-08 22:36:56 -07:00
|
|
|
for k := range table {
|
2009-11-09 13:07:39 -07:00
|
|
|
fmt.Printf("\t%q: %s,\n", k, k)
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
2010-12-07 14:42:54 -07:00
|
|
|
fmt.Print("}\n\n")
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
|
|
|
|
2009-12-15 16:41:46 -07:00
|
|
|
decl := make(sort.StringArray, len(list))
|
|
|
|
ndecl := 0
|
2009-08-27 18:04:23 -06:00
|
|
|
for _, name := range list {
|
2009-08-31 17:43:17 -06:00
|
|
|
if doProps {
|
|
|
|
decl[ndecl] = fmt.Sprintf(
|
|
|
|
"\t%s = _%s;\t// %s is the set of Unicode characters with property %s.\n",
|
2009-11-09 13:07:39 -07:00
|
|
|
name, name, name, name)
|
2009-08-31 17:43:17 -06:00
|
|
|
} else {
|
|
|
|
decl[ndecl] = fmt.Sprintf(
|
|
|
|
"\t%s = _%s;\t// %s is the set of Unicode characters in script %s.\n",
|
2009-11-09 13:07:39 -07:00
|
|
|
name, name, name, name)
|
2009-08-31 17:43:17 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
ndecl++
|
|
|
|
fmt.Printf("var _%s = []Range {\n", name)
|
|
|
|
ranges := foldAdjacent(table[name])
|
2009-08-27 18:04:23 -06:00
|
|
|
for _, s := range ranges {
|
2009-11-09 13:07:39 -07:00
|
|
|
fmt.Printf(format, s.Lo, s.Hi, s.Stride)
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
2010-12-07 14:42:54 -07:00
|
|
|
fmt.Print("}\n\n")
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
decl.Sort()
|
|
|
|
fmt.Println("var (")
|
2009-08-27 19:38:02 -06:00
|
|
|
for _, d := range decl {
|
2009-11-09 13:07:39 -07:00
|
|
|
fmt.Print(d)
|
2009-08-27 19:38:02 -06:00
|
|
|
}
|
2010-12-07 14:42:54 -07:00
|
|
|
fmt.Print(")\n\n")
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
|
|
|
|
2009-08-29 00:05:16 -06:00
|
|
|
const (
|
2009-12-15 16:41:46 -07:00
|
|
|
CaseUpper = 1 << iota
|
|
|
|
CaseLower
|
|
|
|
CaseTitle
|
|
|
|
CaseNone = 0 // must be zero
|
|
|
|
CaseMissing = -1 // character not present; not a valid case state
|
2009-08-29 00:05:16 -06:00
|
|
|
)
|
|
|
|
|
|
|
|
type caseState struct {
|
2009-12-15 16:41:46 -07:00
|
|
|
point int
|
|
|
|
_case int
|
|
|
|
deltaToUpper int
|
|
|
|
deltaToLower int
|
|
|
|
deltaToTitle int
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
// Is d a continuation of the state of c?
|
|
|
|
func (c *caseState) adjacent(d *caseState) bool {
|
|
|
|
if d.point < c.point {
|
2009-11-09 13:07:39 -07:00
|
|
|
c, d = d, c
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
switch {
|
2009-12-15 16:41:46 -07:00
|
|
|
case d.point != c.point+1: // code points not adjacent (shouldn't happen)
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-12-15 16:41:46 -07:00
|
|
|
case d._case != c._case: // different cases
|
2009-11-09 13:07:39 -07:00
|
|
|
return c.upperLowerAdjacent(d)
|
2009-08-29 00:05:16 -06:00
|
|
|
case c._case == CaseNone:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-29 00:05:16 -06:00
|
|
|
case c._case == CaseMissing:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-29 00:05:16 -06:00
|
|
|
case d.deltaToUpper != c.deltaToUpper:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-29 00:05:16 -06:00
|
|
|
case d.deltaToLower != c.deltaToLower:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-29 00:05:16 -06:00
|
|
|
case d.deltaToTitle != c.deltaToTitle:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
return true
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
|
2009-08-30 15:02:42 -06:00
|
|
|
// Is d the same as c, but opposite in upper/lower case? this would make it
|
|
|
|
// an element of an UpperLower sequence.
|
|
|
|
func (c *caseState) upperLowerAdjacent(d *caseState) bool {
|
|
|
|
// check they're a matched case pair. we know they have adjacent values
|
|
|
|
switch {
|
|
|
|
case c._case == CaseUpper && d._case != CaseLower:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-30 15:02:42 -06:00
|
|
|
case c._case == CaseLower && d._case != CaseUpper:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-30 15:02:42 -06:00
|
|
|
}
|
|
|
|
// matched pair (at least in upper/lower). make the order Upper Lower
|
|
|
|
if c._case == CaseLower {
|
2009-11-09 13:07:39 -07:00
|
|
|
c, d = d, c
|
2009-08-30 15:02:42 -06:00
|
|
|
}
|
|
|
|
// for an Upper Lower sequence the deltas have to be in order
|
|
|
|
// c: 0 1 0
|
|
|
|
// d: -1 0 -1
|
|
|
|
switch {
|
|
|
|
case c.deltaToUpper != 0:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-30 15:02:42 -06:00
|
|
|
case c.deltaToLower != 1:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-30 15:02:42 -06:00
|
|
|
case c.deltaToTitle != 0:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-30 15:02:42 -06:00
|
|
|
case d.deltaToUpper != -1:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-30 15:02:42 -06:00
|
|
|
case d.deltaToLower != 0:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-30 15:02:42 -06:00
|
|
|
case d.deltaToTitle != -1:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-30 15:02:42 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
return true
|
2009-08-30 15:02:42 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
// Does this character start an UpperLower sequence?
|
|
|
|
func (c *caseState) isUpperLower() bool {
|
|
|
|
// for an Upper Lower sequence the deltas have to be in order
|
|
|
|
// c: 0 1 0
|
|
|
|
switch {
|
|
|
|
case c.deltaToUpper != 0:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-30 15:02:42 -06:00
|
|
|
case c.deltaToLower != 1:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-30 15:02:42 -06:00
|
|
|
case c.deltaToTitle != 0:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-30 15:02:42 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
return true
|
2009-08-30 15:02:42 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
// Does this character start a LowerUpper sequence?
|
|
|
|
func (c *caseState) isLowerUpper() bool {
|
|
|
|
// for an Upper Lower sequence the deltas have to be in order
|
|
|
|
// c: -1 0 -1
|
|
|
|
switch {
|
|
|
|
case c.deltaToUpper != -1:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-30 15:02:42 -06:00
|
|
|
case c.deltaToLower != 0:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-30 15:02:42 -06:00
|
|
|
case c.deltaToTitle != -1:
|
2009-11-09 13:07:39 -07:00
|
|
|
return false
|
2009-08-30 15:02:42 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
return true
|
2009-08-30 15:02:42 -06:00
|
|
|
}
|
|
|
|
|
2009-08-29 00:05:16 -06:00
|
|
|
func getCaseState(i int) (c *caseState) {
|
2009-12-15 16:41:46 -07:00
|
|
|
c = &caseState{point: i, _case: CaseNone}
|
|
|
|
ch := &chars[i]
|
2009-08-29 00:05:16 -06:00
|
|
|
switch int(ch.codePoint) {
|
|
|
|
case 0:
|
2009-12-15 16:41:46 -07:00
|
|
|
c._case = CaseMissing // Will get NUL wrong but that doesn't matter
|
|
|
|
return
|
2009-08-29 00:05:16 -06:00
|
|
|
case ch.upperCase:
|
2009-11-09 13:07:39 -07:00
|
|
|
c._case = CaseUpper
|
2009-08-29 00:05:16 -06:00
|
|
|
case ch.lowerCase:
|
2009-11-09 13:07:39 -07:00
|
|
|
c._case = CaseLower
|
2009-08-29 00:05:16 -06:00
|
|
|
case ch.titleCase:
|
2009-11-09 13:07:39 -07:00
|
|
|
c._case = CaseTitle
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
if ch.upperCase != 0 {
|
2009-11-09 13:07:39 -07:00
|
|
|
c.deltaToUpper = ch.upperCase - i
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
if ch.lowerCase != 0 {
|
2009-11-09 13:07:39 -07:00
|
|
|
c.deltaToLower = ch.lowerCase - i
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
if ch.titleCase != 0 {
|
2009-11-09 13:07:39 -07:00
|
|
|
c.deltaToTitle = ch.titleCase - i
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
return
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
func printCases() {
|
|
|
|
if !*cases {
|
2009-11-09 13:07:39 -07:00
|
|
|
return
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
if *test {
|
2009-12-15 16:41:46 -07:00
|
|
|
fullCaseTest()
|
|
|
|
return
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
fmt.Printf(
|
2009-12-09 17:54:07 -07:00
|
|
|
"// Generated by running\n"+
|
|
|
|
"// maketables --data=%s\n"+
|
|
|
|
"// DO NOT EDIT\n\n"+
|
|
|
|
"// CaseRanges is the table describing case mappings for all letters with\n"+
|
|
|
|
"// non-self mappings.\n"+
|
|
|
|
"var CaseRanges = _CaseRanges\n"+
|
2009-11-05 19:26:16 -07:00
|
|
|
"var _CaseRanges = []CaseRange {\n",
|
2009-12-15 16:41:46 -07:00
|
|
|
*dataURL)
|
2009-08-29 00:05:16 -06:00
|
|
|
|
2009-12-15 16:41:46 -07:00
|
|
|
var startState *caseState // the start of a run; nil for not active
|
|
|
|
var prevState = &caseState{} // the state of the previous character
|
2009-11-02 12:37:52 -07:00
|
|
|
for i := range chars {
|
2009-12-15 16:41:46 -07:00
|
|
|
state := getCaseState(i)
|
2009-08-29 00:05:16 -06:00
|
|
|
if state.adjacent(prevState) {
|
2009-12-15 16:41:46 -07:00
|
|
|
prevState = state
|
|
|
|
continue
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
// end of run (possibly)
|
2009-12-15 16:41:46 -07:00
|
|
|
printCaseRange(startState, prevState)
|
|
|
|
startState = nil
|
2009-08-29 00:05:16 -06:00
|
|
|
if state._case != CaseMissing && state._case != CaseNone {
|
2009-11-09 13:07:39 -07:00
|
|
|
startState = state
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
prevState = state
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
2010-12-07 14:42:54 -07:00
|
|
|
fmt.Print("}\n")
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
func printCaseRange(lo, hi *caseState) {
|
|
|
|
if lo == nil {
|
2009-11-09 13:07:39 -07:00
|
|
|
return
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
if lo.deltaToUpper == 0 && lo.deltaToLower == 0 && lo.deltaToTitle == 0 {
|
|
|
|
// character represents itself in all cases - no need to mention it
|
2009-11-09 13:07:39 -07:00
|
|
|
return
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
2009-08-30 15:02:42 -06:00
|
|
|
switch {
|
|
|
|
case hi.point > lo.point && lo.isUpperLower():
|
2011-01-31 16:20:44 -07:00
|
|
|
fmt.Printf("\t{0x%04X, 0x%04X, d{UpperLower, UpperLower, UpperLower}},\n",
|
2009-11-09 13:07:39 -07:00
|
|
|
lo.point, hi.point)
|
2009-08-30 15:02:42 -06:00
|
|
|
case hi.point > lo.point && lo.isLowerUpper():
|
2011-02-01 13:47:35 -07:00
|
|
|
logger.Fatalf("LowerUpper sequence: should not happen: U+%04X. If it's real, need to fix To()", lo.point)
|
2011-01-31 16:20:44 -07:00
|
|
|
fmt.Printf("\t{0x%04X, 0x%04X, d{LowerUpper, LowerUpper, LowerUpper}},\n",
|
2009-12-15 16:41:46 -07:00
|
|
|
lo.point, hi.point)
|
2009-08-30 15:02:42 -06:00
|
|
|
default:
|
2011-01-31 16:20:44 -07:00
|
|
|
fmt.Printf("\t{0x%04X, 0x%04X, d{%d, %d, %d}},\n",
|
2009-08-30 15:02:42 -06:00
|
|
|
lo.point, hi.point,
|
2009-11-09 13:07:39 -07:00
|
|
|
lo.deltaToUpper, lo.deltaToLower, lo.deltaToTitle)
|
2009-08-30 15:02:42 -06:00
|
|
|
}
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
// If the cased value in the Char is 0, it means use the rune itself.
|
|
|
|
func caseIt(rune, cased int) int {
|
|
|
|
if cased == 0 {
|
2009-11-09 13:07:39 -07:00
|
|
|
return rune
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
return cased
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
func fullCaseTest() {
|
|
|
|
for i, c := range chars {
|
2009-12-15 16:41:46 -07:00
|
|
|
lower := unicode.ToLower(i)
|
|
|
|
want := caseIt(i, c.lowerCase)
|
2009-08-29 00:05:16 -06:00
|
|
|
if lower != want {
|
2009-11-09 13:07:39 -07:00
|
|
|
fmt.Fprintf(os.Stderr, "lower U+%04X should be U+%04X is U+%04X\n", i, want, lower)
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
upper := unicode.ToUpper(i)
|
|
|
|
want = caseIt(i, c.upperCase)
|
2009-08-29 00:05:16 -06:00
|
|
|
if upper != want {
|
2009-11-09 13:07:39 -07:00
|
|
|
fmt.Fprintf(os.Stderr, "upper U+%04X should be U+%04X is U+%04X\n", i, want, upper)
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
title := unicode.ToTitle(i)
|
|
|
|
want = caseIt(i, c.titleCase)
|
2009-08-29 00:05:16 -06:00
|
|
|
if title != want {
|
2009-11-09 13:07:39 -07:00
|
|
|
fmt.Fprintf(os.Stderr, "title U+%04X should be U+%04X is U+%04X\n", i, want, title)
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|