2009-01-14 15:05:00 -07:00
|
|
|
|
// Copyright 2009 The Go Authors. All rights reserved.
|
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
2011-04-19 17:57:05 -06:00
|
|
|
|
// Package unicode provides data and functions to test some properties of
|
|
|
|
|
// Unicode code points.
|
2009-01-14 15:05:00 -07:00
|
|
|
|
package unicode
|
|
|
|
|
|
2009-08-31 14:01:25 -06:00
|
|
|
|
const (
|
2009-12-15 16:41:46 -07:00
|
|
|
|
MaxRune = 0x10FFFF // Maximum valid Unicode code point.
|
|
|
|
|
ReplacementChar = 0xFFFD // Represents invalid code points.
|
2011-06-03 17:28:27 -06:00
|
|
|
|
MaxASCII = 0x7F // maximum ASCII value.
|
|
|
|
|
MaxLatin1 = 0xFF // maximum Latin-1 value.
|
2009-08-31 14:01:25 -06:00
|
|
|
|
)
|
|
|
|
|
|
2011-05-30 17:58:07 -06:00
|
|
|
|
// RangeTable defines a set of Unicode code points by listing the ranges of
|
|
|
|
|
// code points within the set. The ranges are listed in two slices
|
|
|
|
|
// to save space: a slice of 16-bit ranges and a slice of 32-bit ranges.
|
|
|
|
|
// The two slices must be in sorted order and non-overlapping.
|
2011-05-31 17:49:51 -06:00
|
|
|
|
// Also, R32 should contain only values >= 0x10000 (1<<16).
|
2011-05-30 17:58:07 -06:00
|
|
|
|
type RangeTable struct {
|
|
|
|
|
R16 []Range16
|
|
|
|
|
R32 []Range32
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Range16 represents of a range of 16-bit Unicode code points. The range runs from Lo to Hi
|
|
|
|
|
// inclusive and has the specified stride.
|
|
|
|
|
type Range16 struct {
|
|
|
|
|
Lo uint16
|
|
|
|
|
Hi uint16
|
|
|
|
|
Stride uint16
|
|
|
|
|
}
|
2009-08-31 14:01:25 -06:00
|
|
|
|
|
2011-05-30 17:58:07 -06:00
|
|
|
|
// Range32 represents of a range of Unicode code points and is used when one or
|
2011-07-23 18:51:19 -06:00
|
|
|
|
// more of the values will not fit in 16 bits. The range runs from Lo to Hi
|
2011-05-31 17:49:51 -06:00
|
|
|
|
// inclusive and has the specified stride. Lo and Hi must always be >= 1<<16.
|
2011-05-30 17:58:07 -06:00
|
|
|
|
type Range32 struct {
|
|
|
|
|
Lo uint32
|
|
|
|
|
Hi uint32
|
|
|
|
|
Stride uint32
|
2009-01-14 15:05:00 -07:00
|
|
|
|
}
|
|
|
|
|
|
2010-03-30 18:51:03 -06:00
|
|
|
|
// CaseRange represents a range of Unicode code points for simple (one
|
|
|
|
|
// code point to one code point) case conversion.
|
2009-08-29 00:05:16 -06:00
|
|
|
|
// The range runs from Lo to Hi inclusive, with a fixed stride of 1. Deltas
|
|
|
|
|
// are the number to add to the code point to reach the code point for a
|
|
|
|
|
// different case for that character. They may be negative. If zero, it
|
2009-08-30 15:02:42 -06:00
|
|
|
|
// means the character is in the corresponding case. There is a special
|
|
|
|
|
// case representing sequences of alternating corresponding Upper and Lower
|
2009-08-30 19:17:52 -06:00
|
|
|
|
// pairs. It appears with a fixed Delta of
|
|
|
|
|
// {UpperLower, UpperLower, UpperLower}
|
|
|
|
|
// The constant UpperLower has an otherwise impossible delta value.
|
2009-08-29 00:05:16 -06:00
|
|
|
|
type CaseRange struct {
|
2011-05-31 17:49:51 -06:00
|
|
|
|
Lo uint32
|
|
|
|
|
Hi uint32
|
2009-12-15 16:41:46 -07:00
|
|
|
|
Delta d
|
2009-08-29 00:05:16 -06:00
|
|
|
|
}
|
|
|
|
|
|
2010-03-30 18:51:03 -06:00
|
|
|
|
// SpecialCase represents language-specific case mappings such as Turkish.
|
|
|
|
|
// Methods of SpecialCase customize (by overriding) the standard mappings.
|
|
|
|
|
type SpecialCase []CaseRange
|
|
|
|
|
|
|
|
|
|
//BUG(r): Provide a mechanism for full case folding (those that involve
|
|
|
|
|
// multiple runes in the input or output).
|
|
|
|
|
|
2009-08-29 00:05:16 -06:00
|
|
|
|
// Indices into the Delta arrays inside CaseRanges for case mapping.
|
|
|
|
|
const (
|
2009-12-15 16:41:46 -07:00
|
|
|
|
UpperCase = iota
|
|
|
|
|
LowerCase
|
|
|
|
|
TitleCase
|
|
|
|
|
MaxCase
|
2009-08-29 00:05:16 -06:00
|
|
|
|
)
|
2009-10-07 12:55:06 -06:00
|
|
|
|
|
2009-12-15 16:41:46 -07:00
|
|
|
|
type d [MaxCase]int32 // to make the CaseRanges text shorter
|
2009-08-29 00:05:16 -06:00
|
|
|
|
|
|
|
|
|
// If the Delta field of a CaseRange is UpperLower or LowerUpper, it means
|
|
|
|
|
// this CaseRange represents a sequence of the form (say)
|
|
|
|
|
// Upper Lower Upper Lower.
|
|
|
|
|
const (
|
2009-12-15 16:41:46 -07:00
|
|
|
|
UpperLower = MaxRune + 1 // (Cannot be a valid delta.)
|
2009-08-29 00:05:16 -06:00
|
|
|
|
)
|
|
|
|
|
|
2011-05-30 17:58:07 -06:00
|
|
|
|
// is16 uses binary search to test whether rune is in the specified slice of 16-bit ranges.
|
|
|
|
|
func is16(ranges []Range16, rune uint16) bool {
|
|
|
|
|
// binary search over ranges
|
|
|
|
|
lo := 0
|
|
|
|
|
hi := len(ranges)
|
|
|
|
|
for lo < hi {
|
|
|
|
|
m := lo + (hi-lo)/2
|
|
|
|
|
r := ranges[m]
|
|
|
|
|
if r.Lo <= rune && rune <= r.Hi {
|
2009-12-15 16:41:46 -07:00
|
|
|
|
return (rune-r.Lo)%r.Stride == 0
|
2009-01-14 15:05:00 -07:00
|
|
|
|
}
|
2011-05-30 17:58:07 -06:00
|
|
|
|
if rune < r.Lo {
|
|
|
|
|
hi = m
|
|
|
|
|
} else {
|
|
|
|
|
lo = m + 1
|
|
|
|
|
}
|
2009-01-14 15:05:00 -07:00
|
|
|
|
}
|
2011-05-30 17:58:07 -06:00
|
|
|
|
return false
|
|
|
|
|
}
|
2009-01-14 15:05:00 -07:00
|
|
|
|
|
2011-05-30 17:58:07 -06:00
|
|
|
|
// is32 uses binary search to test whether rune is in the specified slice of 32-bit ranges.
|
|
|
|
|
func is32(ranges []Range32, rune uint32) bool {
|
2009-01-14 15:05:00 -07:00
|
|
|
|
// binary search over ranges
|
2009-12-15 16:41:46 -07:00
|
|
|
|
lo := 0
|
|
|
|
|
hi := len(ranges)
|
2009-01-14 15:05:00 -07:00
|
|
|
|
for lo < hi {
|
2009-12-15 16:41:46 -07:00
|
|
|
|
m := lo + (hi-lo)/2
|
|
|
|
|
r := ranges[m]
|
2009-03-06 04:22:02 -07:00
|
|
|
|
if r.Lo <= rune && rune <= r.Hi {
|
2009-11-09 22:23:52 -07:00
|
|
|
|
return (rune-r.Lo)%r.Stride == 0
|
2009-01-14 15:05:00 -07:00
|
|
|
|
}
|
2009-03-06 04:22:02 -07:00
|
|
|
|
if rune < r.Lo {
|
2009-11-09 13:07:39 -07:00
|
|
|
|
hi = m
|
2009-01-14 15:05:00 -07:00
|
|
|
|
} else {
|
2009-11-09 22:23:52 -07:00
|
|
|
|
lo = m + 1
|
2009-01-14 15:05:00 -07:00
|
|
|
|
}
|
|
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
|
return false
|
2009-01-14 15:05:00 -07:00
|
|
|
|
}
|
|
|
|
|
|
2011-05-30 17:58:07 -06:00
|
|
|
|
// Is tests whether rune is in the specified table of ranges.
|
|
|
|
|
func Is(rangeTab *RangeTable, rune int) bool {
|
|
|
|
|
// common case: rune is ASCII or Latin-1.
|
2011-06-03 17:28:27 -06:00
|
|
|
|
if uint32(rune) <= MaxLatin1 {
|
2011-05-31 17:49:51 -06:00
|
|
|
|
// Only need to check R16, since R32 is always >= 1<<16.
|
2011-05-30 17:58:07 -06:00
|
|
|
|
r16 := uint16(rune)
|
|
|
|
|
for _, r := range rangeTab.R16 {
|
|
|
|
|
if r16 > r.Hi {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
if r16 < r.Lo {
|
|
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
return (r16-r.Lo)%r.Stride == 0
|
|
|
|
|
}
|
|
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
r16 := rangeTab.R16
|
|
|
|
|
if len(r16) > 0 && rune <= int(r16[len(r16)-1].Hi) {
|
|
|
|
|
return is16(r16, uint16(rune))
|
|
|
|
|
}
|
|
|
|
|
r32 := rangeTab.R32
|
|
|
|
|
if len(r32) > 0 && rune >= int(r32[0].Lo) {
|
|
|
|
|
return is32(r32, uint32(rune))
|
|
|
|
|
}
|
|
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
|
2009-08-26 17:01:31 -06:00
|
|
|
|
// IsUpper reports whether the rune is an upper case letter.
|
2009-01-20 15:40:40 -07:00
|
|
|
|
func IsUpper(rune int) bool {
|
2011-06-03 15:46:22 -06:00
|
|
|
|
// See comment in IsGraphic.
|
2011-06-03 17:28:27 -06:00
|
|
|
|
if uint32(rune) <= MaxLatin1 {
|
2011-06-03 15:46:22 -06:00
|
|
|
|
return properties[uint8(rune)]&pLu != 0
|
2009-08-26 17:53:07 -06:00
|
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
|
return Is(Upper, rune)
|
2009-01-14 15:05:00 -07:00
|
|
|
|
}
|
|
|
|
|
|
2009-08-26 17:01:31 -06:00
|
|
|
|
// IsLower reports whether the rune is a lower case letter.
|
|
|
|
|
func IsLower(rune int) bool {
|
2011-06-03 15:46:22 -06:00
|
|
|
|
// See comment in IsGraphic.
|
2011-06-03 17:28:27 -06:00
|
|
|
|
if uint32(rune) <= MaxLatin1 {
|
2011-06-03 15:46:22 -06:00
|
|
|
|
return properties[uint8(rune)]&pLl != 0
|
2009-08-26 17:53:07 -06:00
|
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
|
return Is(Lower, rune)
|
2009-08-26 17:01:31 -06:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// IsTitle reports whether the rune is a title case letter.
|
|
|
|
|
func IsTitle(rune int) bool {
|
2011-06-03 17:28:27 -06:00
|
|
|
|
if rune <= MaxLatin1 {
|
2009-11-09 13:07:39 -07:00
|
|
|
|
return false
|
2009-08-26 17:53:07 -06:00
|
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
|
return Is(Title, rune)
|
2009-08-26 17:01:31 -06:00
|
|
|
|
}
|
|
|
|
|
|
2010-03-30 18:51:03 -06:00
|
|
|
|
// to maps the rune using the specified case mapping.
|
|
|
|
|
func to(_case int, rune int, caseRange []CaseRange) int {
|
2009-08-29 00:05:16 -06:00
|
|
|
|
if _case < 0 || MaxCase <= _case {
|
2009-12-15 16:41:46 -07:00
|
|
|
|
return ReplacementChar // as reasonable an error as any
|
2009-08-29 00:05:16 -06:00
|
|
|
|
}
|
|
|
|
|
// binary search over ranges
|
2009-12-15 16:41:46 -07:00
|
|
|
|
lo := 0
|
2010-03-30 18:51:03 -06:00
|
|
|
|
hi := len(caseRange)
|
2009-08-29 00:05:16 -06:00
|
|
|
|
for lo < hi {
|
2009-12-15 16:41:46 -07:00
|
|
|
|
m := lo + (hi-lo)/2
|
2010-03-30 18:51:03 -06:00
|
|
|
|
r := caseRange[m]
|
2011-05-31 17:49:51 -06:00
|
|
|
|
if int(r.Lo) <= rune && rune <= int(r.Hi) {
|
2009-12-15 16:41:46 -07:00
|
|
|
|
delta := int(r.Delta[_case])
|
2009-08-31 14:01:25 -06:00
|
|
|
|
if delta > MaxRune {
|
2009-08-30 19:17:52 -06:00
|
|
|
|
// In an Upper-Lower sequence, which always starts with
|
|
|
|
|
// an UpperCase letter, the real deltas always look like:
|
|
|
|
|
// {0, 1, 0} UpperCase (Lower is next)
|
|
|
|
|
// {-1, 0, -1} LowerCase (Upper, Title are previous)
|
|
|
|
|
// The characters at even offsets from the beginning of the
|
|
|
|
|
// sequence are upper case; the ones at odd offsets are lower.
|
|
|
|
|
// The correct mapping can be done by clearing or setting the low
|
2009-09-15 10:41:59 -06:00
|
|
|
|
// bit in the sequence offset.
|
2009-08-30 19:17:52 -06:00
|
|
|
|
// The constants UpperCase and TitleCase are even while LowerCase
|
|
|
|
|
// is odd so we take the low bit from _case.
|
2011-05-31 17:49:51 -06:00
|
|
|
|
return int(r.Lo) + ((rune-int(r.Lo))&^1 | _case&1)
|
2009-08-30 15:02:42 -06:00
|
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
|
return rune + delta
|
2009-08-29 00:05:16 -06:00
|
|
|
|
}
|
2011-05-31 17:49:51 -06:00
|
|
|
|
if rune < int(r.Lo) {
|
2009-11-09 13:07:39 -07:00
|
|
|
|
hi = m
|
2009-08-29 00:05:16 -06:00
|
|
|
|
} else {
|
2009-11-09 22:23:52 -07:00
|
|
|
|
lo = m + 1
|
2009-08-29 00:05:16 -06:00
|
|
|
|
}
|
|
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
|
return rune
|
2009-08-29 00:05:16 -06:00
|
|
|
|
}
|
|
|
|
|
|
2010-03-30 18:51:03 -06:00
|
|
|
|
// To maps the rune to the specified case: UpperCase, LowerCase, or TitleCase.
|
|
|
|
|
func To(_case int, rune int) int {
|
|
|
|
|
return to(_case, rune, CaseRanges)
|
|
|
|
|
}
|
|
|
|
|
|
2010-01-07 10:27:24 -07:00
|
|
|
|
// ToUpper maps the rune to upper case.
|
2009-08-29 00:05:16 -06:00
|
|
|
|
func ToUpper(rune int) int {
|
2011-06-03 17:28:27 -06:00
|
|
|
|
if rune <= MaxASCII {
|
2009-08-29 00:05:16 -06:00
|
|
|
|
if 'a' <= rune && rune <= 'z' {
|
2009-11-09 22:23:52 -07:00
|
|
|
|
rune -= 'a' - 'A'
|
2009-08-29 00:05:16 -06:00
|
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
|
return rune
|
2009-08-29 00:05:16 -06:00
|
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
|
return To(UpperCase, rune)
|
2009-08-29 00:05:16 -06:00
|
|
|
|
}
|
|
|
|
|
|
2010-01-07 10:27:24 -07:00
|
|
|
|
// ToLower maps the rune to lower case.
|
2009-08-29 00:05:16 -06:00
|
|
|
|
func ToLower(rune int) int {
|
2011-06-03 17:28:27 -06:00
|
|
|
|
if rune <= MaxASCII {
|
2009-08-29 00:05:16 -06:00
|
|
|
|
if 'A' <= rune && rune <= 'Z' {
|
2009-11-09 22:23:52 -07:00
|
|
|
|
rune += 'a' - 'A'
|
2009-08-29 00:05:16 -06:00
|
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
|
return rune
|
2009-08-29 00:05:16 -06:00
|
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
|
return To(LowerCase, rune)
|
2009-08-29 00:05:16 -06:00
|
|
|
|
}
|
|
|
|
|
|
2010-01-07 10:27:24 -07:00
|
|
|
|
// ToTitle maps the rune to title case.
|
2009-08-29 00:05:16 -06:00
|
|
|
|
func ToTitle(rune int) int {
|
2011-06-03 17:28:27 -06:00
|
|
|
|
if rune <= MaxASCII {
|
2009-12-15 16:41:46 -07:00
|
|
|
|
if 'a' <= rune && rune <= 'z' { // title case is upper case for ASCII
|
2009-11-09 22:23:52 -07:00
|
|
|
|
rune -= 'a' - 'A'
|
2009-08-29 00:05:16 -06:00
|
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
|
return rune
|
2009-08-29 00:05:16 -06:00
|
|
|
|
}
|
2009-12-15 16:41:46 -07:00
|
|
|
|
return To(TitleCase, rune)
|
2009-08-29 00:05:16 -06:00
|
|
|
|
}
|
2010-03-30 18:51:03 -06:00
|
|
|
|
|
|
|
|
|
// ToUpper maps the rune to upper case giving priority to the special mapping.
|
|
|
|
|
func (special SpecialCase) ToUpper(rune int) int {
|
|
|
|
|
r := to(UpperCase, rune, []CaseRange(special))
|
|
|
|
|
if r == rune {
|
|
|
|
|
r = ToUpper(rune)
|
|
|
|
|
}
|
|
|
|
|
return r
|
|
|
|
|
}
|
|
|
|
|
|
2010-03-30 19:33:06 -06:00
|
|
|
|
// ToTitle maps the rune to title case giving priority to the special mapping.
|
2010-03-30 18:51:03 -06:00
|
|
|
|
func (special SpecialCase) ToTitle(rune int) int {
|
|
|
|
|
r := to(TitleCase, rune, []CaseRange(special))
|
|
|
|
|
if r == rune {
|
|
|
|
|
r = ToTitle(rune)
|
|
|
|
|
}
|
|
|
|
|
return r
|
|
|
|
|
}
|
|
|
|
|
|
2010-03-30 19:33:06 -06:00
|
|
|
|
// ToLower maps the rune to lower case giving priority to the special mapping.
|
2010-03-30 18:51:03 -06:00
|
|
|
|
func (special SpecialCase) ToLower(rune int) int {
|
|
|
|
|
r := to(LowerCase, rune, []CaseRange(special))
|
|
|
|
|
if r == rune {
|
|
|
|
|
r = ToLower(rune)
|
|
|
|
|
}
|
|
|
|
|
return r
|
|
|
|
|
}
|
2011-06-16 15:56:25 -06:00
|
|
|
|
|
|
|
|
|
// caseOrbit is defined in tables.go as []foldPair. Right now all the
|
|
|
|
|
// entries fit in uint16, so use uint16. If that changes, compilation
|
|
|
|
|
// will fail (the constants in the composite literal will not fit in uint16)
|
|
|
|
|
// and the types here can change to uint32.
|
|
|
|
|
type foldPair struct {
|
|
|
|
|
From uint16
|
|
|
|
|
To uint16
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// SimpleFold iterates over Unicode code points equivalent under
|
|
|
|
|
// the Unicode-defined simple case folding. Among the code points
|
|
|
|
|
// equivalent to rune (including rune itself), SimpleFold returns the
|
|
|
|
|
// smallest r >= rune if one exists, or else the smallest r >= 0.
|
|
|
|
|
//
|
|
|
|
|
// For example:
|
|
|
|
|
// SimpleFold('A') = 'a'
|
|
|
|
|
// SimpleFold('a') = 'A'
|
|
|
|
|
//
|
|
|
|
|
// SimpleFold('K') = 'k'
|
|
|
|
|
// SimpleFold('k') = '\u212A' (Kelvin symbol, K)
|
|
|
|
|
// SimpleFold('\u212A') = 'K'
|
|
|
|
|
//
|
|
|
|
|
// SimpleFold('1') = '1'
|
|
|
|
|
//
|
|
|
|
|
func SimpleFold(rune int) int {
|
|
|
|
|
// Consult caseOrbit table for special cases.
|
|
|
|
|
lo := 0
|
|
|
|
|
hi := len(caseOrbit)
|
|
|
|
|
for lo < hi {
|
|
|
|
|
m := lo + (hi-lo)/2
|
|
|
|
|
if int(caseOrbit[m].From) < rune {
|
|
|
|
|
lo = m + 1
|
|
|
|
|
} else {
|
|
|
|
|
hi = m
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if lo < len(caseOrbit) && int(caseOrbit[lo].From) == rune {
|
|
|
|
|
return int(caseOrbit[lo].To)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// No folding specified. This is a one- or two-element
|
|
|
|
|
// equivalence class containing rune and ToLower(rune)
|
|
|
|
|
// and ToUpper(rune) if they are different from rune.
|
|
|
|
|
if l := ToLower(rune); l != rune {
|
|
|
|
|
return l
|
|
|
|
|
}
|
|
|
|
|
return ToUpper(rune)
|
|
|
|
|
}
|