2009-08-26 17:01:31 -06:00
|
|
|
// Copyright 2009 The Go Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
|
|
// Unicode table generator.
|
|
|
|
// Data read from the web.
|
|
|
|
|
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bufio";
|
|
|
|
"flag";
|
|
|
|
"fmt";
|
|
|
|
"http";
|
|
|
|
"log";
|
|
|
|
"os";
|
2009-08-27 19:38:02 -06:00
|
|
|
"sort";
|
2009-08-26 17:01:31 -06:00
|
|
|
"strconv";
|
|
|
|
"strings";
|
2009-08-27 18:04:23 -06:00
|
|
|
"regexp";
|
2009-08-27 10:14:32 -06:00
|
|
|
"unicode";
|
2009-08-26 17:01:31 -06:00
|
|
|
)
|
|
|
|
|
2009-08-29 00:05:16 -06:00
|
|
|
func main() {
|
|
|
|
flag.Parse();
|
|
|
|
loadChars(); // always needed
|
|
|
|
printCategories();
|
|
|
|
printScripts();
|
|
|
|
printCases();
|
|
|
|
}
|
|
|
|
|
2009-08-27 18:04:23 -06:00
|
|
|
var dataUrl = flag.String("data", "", "full URL for UnicodeData.txt; defaults to --url/UnicodeData.txt");
|
2009-08-27 10:14:32 -06:00
|
|
|
var url = flag.String("url",
|
2009-08-27 18:04:23 -06:00
|
|
|
"http://www.unicode.org/Public/5.1.0/ucd/",
|
|
|
|
"URL of Unicode database directory")
|
|
|
|
var tablelist = flag.String("tables",
|
2009-08-27 10:14:32 -06:00
|
|
|
"all",
|
2009-08-27 18:04:23 -06:00
|
|
|
"comma-separated list of which tables to generate; can be letter");
|
|
|
|
var scriptlist = flag.String("scripts",
|
|
|
|
"all",
|
|
|
|
"comma-separated list of which script tables to generate");
|
2009-08-29 00:05:16 -06:00
|
|
|
var cases = flag.Bool("cases",
|
|
|
|
true,
|
|
|
|
"generate case tables");
|
2009-08-27 10:14:32 -06:00
|
|
|
var test = flag.Bool("test",
|
|
|
|
false,
|
|
|
|
"test existing tables; can be used to compare web data with package data");
|
2009-08-27 18:04:23 -06:00
|
|
|
var scriptRe *regexp.Regexp
|
2009-08-26 17:01:31 -06:00
|
|
|
|
|
|
|
var die = log.New(os.Stderr, nil, "", log.Lexit|log.Lshortfile);
|
|
|
|
|
2009-08-27 10:14:32 -06:00
|
|
|
var category = map[string] bool{ "letter":true } // Nd Lu etc. letter is a special case
|
|
|
|
|
2009-08-27 18:04:23 -06:00
|
|
|
// UnicodeData.txt has form:
|
2009-08-26 17:01:31 -06:00
|
|
|
// 0037;DIGIT SEVEN;Nd;0;EN;;7;7;7;N;;;;;
|
|
|
|
// 007A;LATIN SMALL LETTER Z;Ll;0;L;;;;;N;;;005A;;005A
|
|
|
|
// See http://www.unicode.org/Public/5.1.0/ucd/UCD.html for full explanation
|
2009-08-29 00:05:16 -06:00
|
|
|
// The fields:
|
2009-08-26 17:01:31 -06:00
|
|
|
const (
|
|
|
|
FCodePoint = iota;
|
|
|
|
FName;
|
|
|
|
FGeneralCategory;
|
|
|
|
FCanonicalCombiningClass;
|
|
|
|
FBidiClass;
|
|
|
|
FDecompositionType;
|
|
|
|
FDecompositionMapping;
|
|
|
|
FNumericType;
|
|
|
|
FNumericValue;
|
|
|
|
FBidiMirrored;
|
|
|
|
FUnicode1Name;
|
|
|
|
FISOComment;
|
|
|
|
FSimpleUppercaseMapping;
|
|
|
|
FSimpleLowercaseMapping;
|
|
|
|
FSimpleTitlecaseMapping;
|
|
|
|
NumField;
|
|
|
|
|
2009-08-27 10:14:32 -06:00
|
|
|
MaxChar = 0x10FFFF; // anything above this shouldn't exist
|
2009-08-26 17:01:31 -06:00
|
|
|
)
|
|
|
|
|
|
|
|
var fieldName = []string{
|
|
|
|
"CodePoint",
|
|
|
|
"Name",
|
|
|
|
"GeneralCategory",
|
|
|
|
"CanonicalCombiningClass",
|
|
|
|
"BidiClass",
|
|
|
|
"DecompositionType",
|
|
|
|
"DecompositionMapping",
|
|
|
|
"NumericType",
|
|
|
|
"NumericValue",
|
|
|
|
"BidiMirrored",
|
|
|
|
"Unicode1Name",
|
|
|
|
"ISOComment",
|
|
|
|
"SimpleUppercaseMapping",
|
|
|
|
"SimpleLowercaseMapping",
|
|
|
|
"SimpleTitlecaseMapping"
|
|
|
|
}
|
|
|
|
|
|
|
|
// This contains only the properties we're interested in.
|
|
|
|
type Char struct {
|
|
|
|
field []string; // debugging only; could be deleted if we take out char.dump()
|
2009-08-29 00:05:16 -06:00
|
|
|
codePoint uint32; // if zero, this index is not a valid code point.
|
2009-08-26 17:01:31 -06:00
|
|
|
category string;
|
2009-08-29 00:05:16 -06:00
|
|
|
upperCase int;
|
|
|
|
lowerCase int;
|
|
|
|
titleCase int;
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
|
2009-08-27 18:04:23 -06:00
|
|
|
// Scripts.txt has form:
|
|
|
|
// A673 ; Cyrillic # Po SLAVONIC ASTERISK
|
|
|
|
// A67C..A67D ; Cyrillic # Mn [2] COMBINING CYRILLIC KAVYKA..COMBINING CYRILLIC PAYEROK
|
|
|
|
// See http://www.unicode.org/Public/5.1.0/ucd/UCD.html for full explanation
|
|
|
|
|
|
|
|
type Script struct {
|
|
|
|
lo, hi uint32; // range of code points
|
|
|
|
script string;
|
|
|
|
}
|
|
|
|
|
2009-08-29 00:05:16 -06:00
|
|
|
var chars = make([]Char, MaxChar+1)
|
2009-08-27 18:04:23 -06:00
|
|
|
var scripts = make(map[string] []Script)
|
2009-08-26 17:01:31 -06:00
|
|
|
|
|
|
|
var lastChar uint32 = 0;
|
|
|
|
|
2009-08-28 12:57:38 -06:00
|
|
|
// In UnicodeData.txt, some ranges are marked like this:
|
2009-08-29 00:05:16 -06:00
|
|
|
// 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
|
|
|
|
// 4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
|
2009-08-28 12:57:38 -06:00
|
|
|
// parseCategory returns a state variable indicating the weirdness.
|
|
|
|
type State int
|
|
|
|
const (
|
|
|
|
SNormal State = iota; // known to be zero for the type
|
|
|
|
SFirst;
|
|
|
|
SLast;
|
2009-08-29 00:05:16 -06:00
|
|
|
SMissing;
|
2009-08-28 12:57:38 -06:00
|
|
|
)
|
|
|
|
|
|
|
|
func parseCategory(line string) (state State) {
|
2009-08-26 17:01:31 -06:00
|
|
|
field := strings.Split(line, ";", -1);
|
|
|
|
if len(field) != NumField {
|
2009-08-27 10:14:32 -06:00
|
|
|
die.Logf("%5s: %d fields (expected %d)\n", line, len(field), NumField);
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
point, err := strconv.Btoui64(field[FCodePoint], 16);
|
|
|
|
if err != nil {
|
|
|
|
die.Log("%.5s...:", err)
|
|
|
|
}
|
|
|
|
lastChar = uint32(point);
|
|
|
|
if point == 0 {
|
|
|
|
return // not interesting and we use 0 as unset
|
|
|
|
}
|
2009-08-29 00:05:16 -06:00
|
|
|
if point > MaxChar {
|
2009-08-26 17:01:31 -06:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
char := &chars[point];
|
|
|
|
char.field=field;
|
|
|
|
if char.codePoint != 0 {
|
|
|
|
die.Logf("point U+%04x reused\n");
|
|
|
|
}
|
|
|
|
char.codePoint = lastChar;
|
|
|
|
char.category = field[FGeneralCategory];
|
2009-08-27 10:14:32 -06:00
|
|
|
category[char.category] = true;
|
2009-08-26 17:01:31 -06:00
|
|
|
switch char.category {
|
|
|
|
case "Nd":
|
|
|
|
// Decimal digit
|
|
|
|
v, err := strconv.Atoi(field[FNumericValue]);
|
|
|
|
if err != nil {
|
|
|
|
die.Log("U+%04x: bad numeric field: %s", point, err);
|
|
|
|
}
|
|
|
|
case "Lu":
|
|
|
|
char.letter(field[FCodePoint], field[FSimpleLowercaseMapping], field[FSimpleTitlecaseMapping]);
|
|
|
|
case "Ll":
|
|
|
|
char.letter(field[FSimpleUppercaseMapping], field[FCodePoint], field[FSimpleTitlecaseMapping]);
|
|
|
|
case "Lt":
|
|
|
|
char.letter(field[FSimpleUppercaseMapping], field[FSimpleLowercaseMapping], field[FCodePoint]);
|
|
|
|
case "Lm", "Lo":
|
|
|
|
char.letter(field[FSimpleUppercaseMapping], field[FSimpleLowercaseMapping], field[FSimpleTitlecaseMapping]);
|
|
|
|
}
|
2009-08-28 12:57:38 -06:00
|
|
|
switch {
|
|
|
|
case strings.Index(field[FName], ", First>") > 0:
|
|
|
|
state = SFirst
|
|
|
|
case strings.Index(field[FName], ", Last>") > 0:
|
|
|
|
state = SLast
|
|
|
|
}
|
|
|
|
return
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
func (char *Char) dump(s string) {
|
|
|
|
fmt.Print(s, " ");
|
|
|
|
for i:=0;i<len(char.field);i++ {
|
|
|
|
fmt.Printf("%s:%q ", fieldName[i], char.field[i]);
|
|
|
|
}
|
|
|
|
fmt.Print("\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
func (char *Char) letter(u, l, t string) {
|
|
|
|
char.upperCase = char.letterValue(u, "U");
|
|
|
|
char.lowerCase = char.letterValue(l, "L");
|
|
|
|
char.titleCase = char.letterValue(t, "T");
|
|
|
|
}
|
|
|
|
|
2009-08-29 00:05:16 -06:00
|
|
|
func (char *Char) letterValue(s string, cas string) int {
|
2009-08-26 17:01:31 -06:00
|
|
|
if s == "" {
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
v, err := strconv.Btoui64(s, 16);
|
|
|
|
if err != nil {
|
|
|
|
char.dump(cas);
|
|
|
|
die.Logf("U+%04x: bad letter(%s): %s", char.codePoint, s, err)
|
|
|
|
}
|
2009-08-29 00:05:16 -06:00
|
|
|
return int(v)
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
|
2009-08-27 10:14:32 -06:00
|
|
|
func allCategories() []string {
|
|
|
|
a := make([]string, len(category));
|
|
|
|
i := 0;
|
|
|
|
for k := range category {
|
|
|
|
a[i] = k;
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
return a;
|
|
|
|
}
|
|
|
|
|
2009-08-27 18:04:23 -06:00
|
|
|
func allScripts() []string {
|
|
|
|
a := make([]string, len(scripts));
|
|
|
|
i := 0;
|
|
|
|
for k := range scripts {
|
|
|
|
a[i] = k;
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
return a;
|
|
|
|
}
|
|
|
|
|
2009-08-27 10:14:32 -06:00
|
|
|
// Extract the version number from the URL
|
|
|
|
func version() string {
|
|
|
|
// Break on slashes and look for the first numeric field
|
|
|
|
fields := strings.Split(*url, "/", 0);
|
|
|
|
for _, f := range fields {
|
|
|
|
if len(f) > 0 && '0' <= f[0] && f[0] <= '9' {
|
|
|
|
return f
|
|
|
|
}
|
|
|
|
}
|
|
|
|
die.Log("unknown version");
|
|
|
|
return "Unknown";
|
|
|
|
}
|
|
|
|
|
|
|
|
func letterOp(code int) bool {
|
|
|
|
switch chars[code].category {
|
|
|
|
case "Lu", "Ll", "Lt", "Lm", "Lo":
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2009-08-29 00:05:16 -06:00
|
|
|
func loadChars() {
|
2009-08-27 18:04:23 -06:00
|
|
|
if *dataUrl == "" {
|
|
|
|
flag.Set("data", *url + "UnicodeData.txt");
|
|
|
|
}
|
|
|
|
resp, _, err := http.Get(*dataUrl);
|
2009-08-26 17:01:31 -06:00
|
|
|
if err != nil {
|
|
|
|
die.Log(err);
|
|
|
|
}
|
2009-08-27 10:14:32 -06:00
|
|
|
if resp.StatusCode != 200 {
|
2009-08-27 19:38:02 -06:00
|
|
|
die.Log("bad GET status for UnicodeData.txt", resp.Status);
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
2009-08-26 17:01:31 -06:00
|
|
|
input := bufio.NewReader(resp.Body);
|
2009-08-28 12:57:38 -06:00
|
|
|
var first uint32 = 0;
|
2009-08-26 17:01:31 -06:00
|
|
|
for {
|
2009-08-27 18:04:23 -06:00
|
|
|
line, err := input.ReadString('\n');
|
2009-08-26 17:01:31 -06:00
|
|
|
if err != nil {
|
|
|
|
if err == os.EOF {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
die.Log(err);
|
|
|
|
}
|
2009-08-28 12:57:38 -06:00
|
|
|
switch parseCategory(line[0:len(line)-1]) {
|
|
|
|
case SNormal:
|
|
|
|
if first != 0 {
|
|
|
|
die.Logf("bad state normal at U+%04X", lastChar)
|
|
|
|
}
|
|
|
|
case SFirst:
|
|
|
|
if first != 0 {
|
|
|
|
die.Logf("bad state first at U+%04X", lastChar)
|
|
|
|
}
|
|
|
|
first = lastChar
|
|
|
|
case SLast:
|
|
|
|
if first == 0 {
|
|
|
|
die.Logf("bad state last at U+%04X", lastChar)
|
|
|
|
}
|
|
|
|
for i := first+1; i <= lastChar; i++ {
|
|
|
|
chars[i] = chars[first];
|
|
|
|
chars[i].codePoint = i;
|
|
|
|
}
|
|
|
|
first = 0
|
|
|
|
}
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
resp.Body.Close();
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
func printCategories() {
|
|
|
|
if *tablelist == "" {
|
|
|
|
return
|
|
|
|
}
|
2009-08-27 10:14:32 -06:00
|
|
|
// Find out which categories to dump
|
2009-08-27 18:04:23 -06:00
|
|
|
list := strings.Split(*tablelist, ",", 0);
|
|
|
|
if *tablelist == "all" {
|
|
|
|
list = allCategories()
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
|
|
|
if *test {
|
2009-08-27 18:04:23 -06:00
|
|
|
fullCategoryTest(list);
|
2009-08-27 10:14:32 -06:00
|
|
|
return
|
|
|
|
}
|
2009-08-26 17:01:31 -06:00
|
|
|
fmt.Printf(
|
|
|
|
"// Generated by running\n"
|
2009-08-29 00:05:16 -06:00
|
|
|
"// maketables --tables=%s --data=%s\n"
|
2009-08-26 17:01:31 -06:00
|
|
|
"// DO NOT EDIT\n\n"
|
2009-08-27 10:14:32 -06:00
|
|
|
"package unicode\n\n",
|
2009-08-27 18:04:23 -06:00
|
|
|
*tablelist,
|
2009-08-29 00:05:16 -06:00
|
|
|
*dataUrl
|
2009-08-26 17:01:31 -06:00
|
|
|
);
|
2009-08-27 10:14:32 -06:00
|
|
|
|
|
|
|
fmt.Println("// Version is the Unicode edition from which the tables are derived.");
|
2009-08-27 10:58:43 -06:00
|
|
|
fmt.Printf("const Version = %q\n\n", version());
|
2009-08-27 10:14:32 -06:00
|
|
|
|
2009-08-27 18:04:23 -06:00
|
|
|
if *tablelist == "all" {
|
|
|
|
fmt.Println("// Categories is the set of Unicode data tables.");
|
|
|
|
fmt.Println("var Categories = map[string] []Range {");
|
2009-08-27 10:14:32 -06:00
|
|
|
for k, _ := range category {
|
|
|
|
fmt.Printf("\t%q: %s,\n", k, k);
|
|
|
|
}
|
|
|
|
fmt.Printf("}\n\n");
|
|
|
|
}
|
|
|
|
|
2009-08-27 19:38:02 -06:00
|
|
|
decl := make(sort.StringArray, len(list));
|
|
|
|
ndecl := 0;
|
2009-08-27 10:14:32 -06:00
|
|
|
for _, name := range list {
|
|
|
|
if _, ok := category[name]; !ok {
|
|
|
|
die.Log("unknown category", name);
|
|
|
|
}
|
|
|
|
// We generate an UpperCase name to serve as concise documentation and an _UnderScored
|
|
|
|
// name to store the data. This stops godoc dumping all the tables but keeps them
|
|
|
|
// available to clients.
|
|
|
|
// Cases deserving special comments
|
2009-08-27 19:38:02 -06:00
|
|
|
varDecl := "";
|
2009-08-27 10:14:32 -06:00
|
|
|
switch name {
|
2009-08-27 19:38:02 -06:00
|
|
|
case "letter":
|
|
|
|
varDecl = "\tLetter = letter; // Letter is the set of Unicode letters.\n";
|
2009-08-27 10:14:32 -06:00
|
|
|
case "Nd":
|
2009-08-27 19:38:02 -06:00
|
|
|
varDecl = "\tDigit = _Nd; // Digit is the set of Unicode characters with the \"decimal digit\" property.\n";
|
2009-08-27 10:14:32 -06:00
|
|
|
case "Lu":
|
2009-08-27 19:38:02 -06:00
|
|
|
varDecl = "\tUpper = _Lu; // Upper is the set of Unicode upper case letters.\n";
|
2009-08-27 10:14:32 -06:00
|
|
|
case "Ll":
|
2009-08-27 19:38:02 -06:00
|
|
|
varDecl = "\tLower = _Ll; // Lower is the set of Unicode lower case letters.\n";
|
2009-08-27 10:14:32 -06:00
|
|
|
case "Lt":
|
2009-08-27 19:38:02 -06:00
|
|
|
varDecl = "\tTitle = _Lt; // Title is the set of Unicode title case letters.\n";
|
|
|
|
}
|
|
|
|
if name != "letter" {
|
|
|
|
varDecl += fmt.Sprintf(
|
|
|
|
"\t%s = _%s; // %s is the set of Unicode characters in category %s.\n",
|
|
|
|
name, name, name, name
|
|
|
|
);
|
|
|
|
}
|
|
|
|
decl[ndecl] = varDecl;
|
|
|
|
ndecl++;
|
|
|
|
if name == "letter" { // special case
|
|
|
|
dumpRange(
|
|
|
|
"var letter = []Range {\n",
|
|
|
|
letterOp
|
|
|
|
);
|
|
|
|
continue;
|
2009-08-27 10:14:32 -06:00
|
|
|
}
|
2009-08-26 17:01:31 -06:00
|
|
|
dumpRange(
|
2009-08-27 19:38:02 -06:00
|
|
|
fmt.Sprintf("var _%s = []Range {\n", name),
|
|
|
|
func(code int) bool { return chars[code].category == name }
|
2009-08-26 17:01:31 -06:00
|
|
|
);
|
|
|
|
}
|
2009-08-27 19:38:02 -06:00
|
|
|
decl.Sort();
|
|
|
|
fmt.Println("var (");
|
|
|
|
for _, d := range decl {
|
|
|
|
fmt.Print(d);
|
|
|
|
}
|
|
|
|
fmt.Println(")\n");
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
type Op func(code int) bool
|
2009-08-27 18:04:23 -06:00
|
|
|
const format = "\tRange{0x%04x, 0x%04x, %d},\n";
|
2009-08-26 17:01:31 -06:00
|
|
|
|
2009-08-27 19:38:02 -06:00
|
|
|
func dumpRange(header string, inCategory Op) {
|
2009-08-26 17:01:31 -06:00
|
|
|
fmt.Print(header);
|
|
|
|
next := 0;
|
|
|
|
// one Range for each iteration
|
|
|
|
for {
|
|
|
|
// look for start of range
|
|
|
|
for next < len(chars) && !inCategory(next) {
|
|
|
|
next++
|
|
|
|
}
|
|
|
|
if next >= len(chars) {
|
|
|
|
// no characters remain
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
// start of range
|
|
|
|
lo := next;
|
|
|
|
hi := next;
|
|
|
|
stride := 1;
|
|
|
|
// accept lo
|
|
|
|
next++;
|
|
|
|
// look for another character to set the stride
|
|
|
|
for next < len(chars) && !inCategory(next) {
|
|
|
|
next++
|
|
|
|
}
|
|
|
|
if next >= len(chars) {
|
|
|
|
// no more characters
|
|
|
|
fmt.Printf(format, lo, hi, stride);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
// set stride
|
|
|
|
stride = next - lo;
|
|
|
|
// check for length of run. next points to first jump in stride
|
|
|
|
for i := next; i < len(chars); i++ {
|
|
|
|
if inCategory(i) == (((i-lo)%stride) == 0) {
|
|
|
|
// accept
|
|
|
|
if inCategory(i) {
|
|
|
|
hi = i
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// no more characters in this run
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fmt.Printf(format, lo, hi, stride);
|
|
|
|
// next range: start looking where this range ends
|
|
|
|
next = hi + 1;
|
|
|
|
}
|
2009-08-27 19:38:02 -06:00
|
|
|
fmt.Print("}\n\n");
|
2009-08-26 17:01:31 -06:00
|
|
|
}
|
2009-08-27 10:14:32 -06:00
|
|
|
|
2009-08-27 18:04:23 -06:00
|
|
|
func fullCategoryTest(list []string) {
|
2009-08-27 10:14:32 -06:00
|
|
|
for _, name := range list {
|
|
|
|
if _, ok := category[name]; !ok {
|
|
|
|
die.Log("unknown category", name);
|
|
|
|
}
|
2009-08-27 18:04:23 -06:00
|
|
|
r, ok := unicode.Categories[name];
|
2009-08-27 10:14:32 -06:00
|
|
|
if !ok {
|
|
|
|
die.Log("unknown table", name);
|
|
|
|
}
|
|
|
|
if name == "letter" {
|
|
|
|
verifyRange(name, letterOp, r);
|
|
|
|
} else {
|
|
|
|
verifyRange(
|
|
|
|
name,
|
|
|
|
func(code int) bool { return chars[code].category == name },
|
|
|
|
r
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func verifyRange(name string, inCategory Op, table []unicode.Range) {
|
|
|
|
for i, c := range chars {
|
|
|
|
web := inCategory(i);
|
|
|
|
pkg := unicode.Is(table, i);
|
|
|
|
if web != pkg {
|
|
|
|
fmt.Fprintf(os.Stderr, "%s: U+%04X: web=%t pkg=%t\n", name, i, web, pkg);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2009-08-27 18:04:23 -06:00
|
|
|
|
|
|
|
func parseScript(line string) {
|
|
|
|
comment := strings.Index(line, "#");
|
|
|
|
if comment >= 0 {
|
|
|
|
line = line[0:comment]
|
|
|
|
}
|
|
|
|
line = strings.TrimSpaceASCII(line);
|
|
|
|
if len(line) == 0 {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
field := strings.Split(line, ";", -1);
|
|
|
|
if len(field) != 2 {
|
|
|
|
die.Logf("%s: %d fields (expected 2)\n", line, len(field));
|
|
|
|
}
|
|
|
|
matches := scriptRe.MatchStrings(line);
|
|
|
|
if len(matches) != 4 {
|
|
|
|
die.Logf("%s: %d matches (expected 3)\n", line, len(matches));
|
|
|
|
}
|
|
|
|
lo, err := strconv.Btoui64(matches[1], 16);
|
|
|
|
if err != nil {
|
|
|
|
die.Log("%.5s...:", err)
|
|
|
|
}
|
|
|
|
hi := lo;
|
|
|
|
if len(matches[2]) > 2 { // ignore leading ..
|
|
|
|
hi, err = strconv.Btoui64(matches[2][2:len(matches[2])], 16);
|
|
|
|
if err != nil {
|
|
|
|
die.Log("%.5s...:", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
name := matches[3];
|
|
|
|
s, ok := scripts[name];
|
|
|
|
if len(s) == cap(s) {
|
|
|
|
ns := make([]Script, len(s), len(s)+100);
|
|
|
|
for i, sc := range s {
|
|
|
|
ns[i] = sc
|
|
|
|
}
|
|
|
|
s = ns;
|
|
|
|
}
|
|
|
|
s = s[0:len(s)+1];
|
|
|
|
s[len(s)-1] = Script{ uint32(lo), uint32(hi), name };
|
|
|
|
scripts[name] = s;
|
|
|
|
}
|
|
|
|
|
|
|
|
func printScripts() {
|
2009-08-29 00:05:16 -06:00
|
|
|
if *scriptlist == "" {
|
|
|
|
return
|
|
|
|
}
|
2009-08-27 18:04:23 -06:00
|
|
|
var err os.Error;
|
|
|
|
scriptRe, err = regexp.Compile(`([0-9A-F]+)(\.\.[0-9A-F]+)? +; ([A-Za-z_]+)`);
|
|
|
|
if err != nil {
|
|
|
|
die.Log("re error:", err)
|
|
|
|
}
|
|
|
|
resp, _, err := http.Get(*url + "Scripts.txt");
|
|
|
|
if err != nil {
|
|
|
|
die.Log(err);
|
|
|
|
}
|
|
|
|
if resp.StatusCode != 200 {
|
|
|
|
die.Log("bad GET status for Scripts.txt", resp.Status);
|
|
|
|
}
|
|
|
|
input := bufio.NewReader(resp.Body);
|
|
|
|
for {
|
|
|
|
line, err := input.ReadString('\n');
|
|
|
|
if err != nil {
|
|
|
|
if err == os.EOF {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
die.Log(err);
|
|
|
|
}
|
|
|
|
parseScript(line[0:len(line)-1]);
|
|
|
|
}
|
|
|
|
resp.Body.Close();
|
|
|
|
|
|
|
|
// Find out which scripts to dump
|
|
|
|
list := strings.Split(*scriptlist, ",", 0);
|
|
|
|
if *scriptlist == "all" {
|
|
|
|
list = allScripts();
|
|
|
|
}
|
|
|
|
if *test {
|
|
|
|
fullScriptTest(list);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
fmt.Printf(
|
|
|
|
"// Generated by running\n"
|
|
|
|
"// maketables --scripts=%s --url=%s\n"
|
|
|
|
"// DO NOT EDIT\n\n",
|
|
|
|
*scriptlist,
|
|
|
|
*url
|
|
|
|
);
|
|
|
|
if *scriptlist == "all" {
|
|
|
|
fmt.Println("// Scripts is the set of Unicode script tables.");
|
|
|
|
fmt.Println("var Scripts = map[string] []Range {");
|
|
|
|
for k, _ := range scripts {
|
|
|
|
fmt.Printf("\t%q: %s,\n", k, k);
|
|
|
|
}
|
|
|
|
fmt.Printf("}\n\n");
|
|
|
|
}
|
|
|
|
|
2009-08-27 19:38:02 -06:00
|
|
|
decl := make(sort.StringArray, len(list));
|
|
|
|
ndecl := 0;
|
2009-08-27 18:04:23 -06:00
|
|
|
for _, name := range list {
|
2009-08-27 19:38:02 -06:00
|
|
|
decl[ndecl] = fmt.Sprintf(
|
|
|
|
"\t%s = _%s;\t// %s is the set of Unicode characters in script %s.\n",
|
|
|
|
name, name, name, name
|
2009-08-27 18:04:23 -06:00
|
|
|
);
|
2009-08-27 19:38:02 -06:00
|
|
|
ndecl++;
|
|
|
|
fmt.Printf("var _%s = []Range {\n", name);
|
2009-08-27 18:04:23 -06:00
|
|
|
ranges := foldAdjacent(scripts[name]);
|
|
|
|
for _, s := range ranges {
|
|
|
|
fmt.Printf(format, s.Lo, s.Hi, s.Stride);
|
|
|
|
}
|
|
|
|
fmt.Printf("}\n\n");
|
|
|
|
}
|
2009-08-27 19:38:02 -06:00
|
|
|
decl.Sort();
|
|
|
|
fmt.Println("var (");
|
|
|
|
for _, d := range decl {
|
|
|
|
fmt.Print(d);
|
|
|
|
}
|
|
|
|
fmt.Println(")\n");
|
2009-08-27 18:04:23 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
// The script tables have a lot of adjacent elements. Fold them together.
|
|
|
|
func foldAdjacent(r []Script) []unicode.Range {
|
|
|
|
s := make([]unicode.Range, 0, len(r));
|
|
|
|
j := 0;
|
|
|
|
for i := 0; i < len(r); i++ {
|
|
|
|
if j>0 && int(r[i].lo) == s[j-1].Hi+1 {
|
|
|
|
s[j-1].Hi = int(r[i].hi);
|
|
|
|
} else {
|
|
|
|
s = s[0:j+1];
|
|
|
|
s[j] = unicode.Range{int(r[i].lo), int(r[i].hi), 1};
|
|
|
|
j++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
func fullScriptTest(list []string) {
|
|
|
|
for _, name := range list {
|
|
|
|
if _, ok := scripts[name]; !ok {
|
|
|
|
die.Log("unknown script", name);
|
|
|
|
}
|
|
|
|
r, ok := unicode.Scripts[name];
|
|
|
|
if !ok {
|
|
|
|
die.Log("unknown table", name);
|
|
|
|
}
|
|
|
|
for _, script := range scripts[name] {
|
|
|
|
for r := script.lo; r <= script.hi; r++ {
|
|
|
|
if !unicode.Is(unicode.Scripts[name], int(r)) {
|
|
|
|
fmt.Fprintf(os.Stderr, "U+%04X: not in script %s\n", r, name);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2009-08-29 00:05:16 -06:00
|
|
|
|
|
|
|
const (
|
|
|
|
CaseUpper = 1 << iota;
|
|
|
|
CaseLower;
|
|
|
|
CaseTitle;
|
|
|
|
CaseNone = 0; // must be zero
|
|
|
|
CaseMissing = -1; // character not present; not a valid case state
|
|
|
|
)
|
|
|
|
|
|
|
|
type caseState struct {
|
|
|
|
point int;
|
|
|
|
_case int;
|
|
|
|
deltaToUpper int;
|
|
|
|
deltaToLower int;
|
|
|
|
deltaToTitle int;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Is d a continuation of the state of c?
|
|
|
|
func (c *caseState) adjacent(d *caseState) bool {
|
|
|
|
if d.point < c.point {
|
2009-08-30 15:02:42 -06:00
|
|
|
c, d = d, c
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
switch {
|
2009-08-30 15:02:42 -06:00
|
|
|
case d.point != c.point+1: // code points not adjacent (shouldn't happen)
|
2009-08-29 00:05:16 -06:00
|
|
|
return false
|
2009-08-30 15:02:42 -06:00
|
|
|
case d._case != c._case: // different cases
|
|
|
|
return c.upperLowerAdjacent(d);
|
2009-08-29 00:05:16 -06:00
|
|
|
case c._case == CaseNone:
|
|
|
|
return false
|
|
|
|
case c._case == CaseMissing:
|
|
|
|
return false
|
|
|
|
case d.deltaToUpper != c.deltaToUpper:
|
|
|
|
return false
|
|
|
|
case d.deltaToLower != c.deltaToLower:
|
|
|
|
return false
|
|
|
|
case d.deltaToTitle != c.deltaToTitle:
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2009-08-30 15:02:42 -06:00
|
|
|
// Is d the same as c, but opposite in upper/lower case? this would make it
|
|
|
|
// an element of an UpperLower sequence.
|
|
|
|
func (c *caseState) upperLowerAdjacent(d *caseState) bool {
|
|
|
|
// check they're a matched case pair. we know they have adjacent values
|
|
|
|
switch {
|
|
|
|
case c._case == CaseUpper && d._case != CaseLower:
|
|
|
|
return false
|
|
|
|
case c._case == CaseLower && d._case != CaseUpper:
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
// matched pair (at least in upper/lower). make the order Upper Lower
|
|
|
|
if c._case == CaseLower {
|
|
|
|
c, d = d, c
|
|
|
|
}
|
|
|
|
// for an Upper Lower sequence the deltas have to be in order
|
|
|
|
// c: 0 1 0
|
|
|
|
// d: -1 0 -1
|
|
|
|
switch {
|
|
|
|
case c.deltaToUpper != 0:
|
|
|
|
return false
|
|
|
|
case c.deltaToLower != 1:
|
|
|
|
return false
|
|
|
|
case c.deltaToTitle != 0:
|
|
|
|
return false
|
|
|
|
case d.deltaToUpper != -1:
|
|
|
|
return false
|
|
|
|
case d.deltaToLower != 0:
|
|
|
|
return false
|
|
|
|
case d.deltaToTitle != -1:
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
// Does this character start an UpperLower sequence?
|
|
|
|
func (c *caseState) isUpperLower() bool {
|
|
|
|
// for an Upper Lower sequence the deltas have to be in order
|
|
|
|
// c: 0 1 0
|
|
|
|
switch {
|
|
|
|
case c.deltaToUpper != 0:
|
|
|
|
return false
|
|
|
|
case c.deltaToLower != 1:
|
|
|
|
return false
|
|
|
|
case c.deltaToTitle != 0:
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
// Does this character start a LowerUpper sequence?
|
|
|
|
func (c *caseState) isLowerUpper() bool {
|
|
|
|
// for an Upper Lower sequence the deltas have to be in order
|
|
|
|
// c: -1 0 -1
|
|
|
|
switch {
|
|
|
|
case c.deltaToUpper != -1:
|
|
|
|
return false
|
|
|
|
case c.deltaToLower != 0:
|
|
|
|
return false
|
|
|
|
case c.deltaToTitle != -1:
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2009-08-29 00:05:16 -06:00
|
|
|
func getCaseState(i int) (c *caseState) {
|
|
|
|
c = &caseState{ point: i, _case: CaseNone };
|
|
|
|
ch := &chars[i];
|
|
|
|
switch int(ch.codePoint) {
|
|
|
|
case 0:
|
|
|
|
c._case = CaseMissing; // Will get NUL wrong but that doesn't matter
|
|
|
|
return;
|
|
|
|
case ch.upperCase:
|
|
|
|
c._case = CaseUpper;
|
|
|
|
case ch.lowerCase:
|
|
|
|
c._case = CaseLower;
|
|
|
|
case ch.titleCase:
|
|
|
|
c._case = CaseTitle;
|
|
|
|
}
|
|
|
|
if ch.upperCase != 0 {
|
|
|
|
c.deltaToUpper = ch.upperCase - i
|
|
|
|
}
|
|
|
|
if ch.lowerCase != 0 {
|
|
|
|
c.deltaToLower = ch.lowerCase - i
|
|
|
|
}
|
|
|
|
if ch.titleCase != 0 {
|
|
|
|
c.deltaToTitle = ch.titleCase - i
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
func printCases() {
|
|
|
|
if !*cases {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if *test {
|
|
|
|
fullCaseTest();
|
|
|
|
return
|
|
|
|
}
|
|
|
|
fmt.Printf(
|
|
|
|
"// Generated by running\n"
|
|
|
|
"// maketables --data=%s\n"
|
|
|
|
"// DO NOT EDIT\n\n"
|
|
|
|
"// CaseRanges is the table describing case mappings for all letters with\n"
|
|
|
|
"// non-self mappings.\n"
|
|
|
|
"var CaseRanges = _CaseRanges\n"
|
|
|
|
"var _CaseRanges = []CaseRange {\n",
|
|
|
|
*dataUrl
|
|
|
|
);
|
|
|
|
|
|
|
|
var startState *caseState; // the start of a run; nil for not active
|
|
|
|
var prevState = &caseState{}; // the state of the previous character
|
|
|
|
for i, c := range chars {
|
|
|
|
state := getCaseState(i);
|
|
|
|
if state.adjacent(prevState) {
|
|
|
|
prevState = state;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
// end of run (possibly)
|
|
|
|
printCaseRange(startState, prevState);
|
|
|
|
startState = nil;
|
|
|
|
if state._case != CaseMissing && state._case != CaseNone {
|
|
|
|
startState = state;
|
|
|
|
}
|
|
|
|
prevState = state;
|
|
|
|
}
|
|
|
|
fmt.Printf("}\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
func printCaseRange(lo, hi *caseState) {
|
|
|
|
if lo == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if lo.deltaToUpper == 0 && lo.deltaToLower == 0 && lo.deltaToTitle == 0 {
|
|
|
|
// character represents itself in all cases - no need to mention it
|
|
|
|
return
|
|
|
|
}
|
2009-08-30 15:02:42 -06:00
|
|
|
switch {
|
|
|
|
case hi.point > lo.point && lo.isUpperLower():
|
|
|
|
fmt.Printf("\tCaseRange{0x%04X, 0x%04X, d{UpperLower, UpperLower, UpperLower}},\n",
|
|
|
|
lo.point, hi.point)
|
|
|
|
case hi.point > lo.point && lo.isLowerUpper():
|
2009-08-30 19:17:52 -06:00
|
|
|
die.Log("LowerUpper sequence: should not happen: U+%04X. If it's real, need to fix To()", lo.point);
|
2009-08-30 15:02:42 -06:00
|
|
|
fmt.Printf("\tCaseRange{0x%04X, 0x%04X, d{LowerUpper, LowerUpper, LowerUpper}},\n",
|
|
|
|
lo.point, hi.point)
|
|
|
|
default:
|
|
|
|
fmt.Printf("\tCaseRange{0x%04X, 0x%04X, d{%d, %d, %d}},\n",
|
|
|
|
lo.point, hi.point,
|
|
|
|
lo.deltaToUpper, lo.deltaToLower, lo.deltaToTitle)
|
|
|
|
}
|
2009-08-29 00:05:16 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
// If the cased value in the Char is 0, it means use the rune itself.
|
|
|
|
func caseIt(rune, cased int) int {
|
|
|
|
if cased == 0 {
|
|
|
|
return rune
|
|
|
|
}
|
|
|
|
return cased
|
|
|
|
}
|
|
|
|
|
|
|
|
func fullCaseTest() {
|
|
|
|
for i, c := range chars {
|
|
|
|
lower := unicode.ToLower(i);
|
|
|
|
want := caseIt(i, c.lowerCase);
|
|
|
|
if lower != want {
|
|
|
|
fmt.Fprintf(os.Stderr, "lower U+%04X should be U+%04X is U+%04X\n", i, want, lower);
|
|
|
|
}
|
|
|
|
upper := unicode.ToUpper(i);
|
|
|
|
want = caseIt(i, c.upperCase);
|
|
|
|
if upper != want {
|
|
|
|
fmt.Fprintf(os.Stderr, "upper U+%04X should be U+%04X is U+%04X\n", i, want, upper);
|
|
|
|
}
|
|
|
|
title := unicode.ToTitle(i);
|
|
|
|
want = caseIt(i, c.titleCase);
|
|
|
|
if title != want {
|
|
|
|
fmt.Fprintf(os.Stderr, "title U+%04X should be U+%04X is U+%04X\n", i, want, title);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|