gemini-browser

forminfo.go (8913B)

---

 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 package norm
 
 import "encoding/binary"
 
 // This file contains Form-specific logic and wrappers for data in tables.go.
 
 // Rune info is stored in a separate trie per composing form. A composing form
 // and its corresponding decomposing form share the same trie.  Each trie maps
 // a rune to a uint16. The values take two forms.  For v >= 0x8000:
 //   bits
 //   15:    1 (inverse of NFD_QC bit of qcInfo)
 //   13..7: qcInfo (see below). isYesD is always true (no decomposition).
 //    6..0: ccc (compressed CCC value).
 // For v < 0x8000, the respective rune has a decomposition and v is an index
 // into a byte array of UTF-8 decomposition sequences and additional info and
 // has the form:
 //    <header> <decomp_byte>* [<tccc> [<lccc>]]
 // The header contains the number of bytes in the decomposition (excluding this
 // length byte). The two most significant bits of this length byte correspond
 // to bit 5 and 4 of qcInfo (see below).  The byte sequence itself starts at v+1.
 // The byte sequence is followed by a trailing and leading CCC if the values
 // for these are not zero.  The value of v determines which ccc are appended
 // to the sequences.  For v < firstCCC, there are none, for v >= firstCCC,
 // the sequence is followed by a trailing ccc, and for v >= firstLeadingCC
 // there is an additional leading ccc. The value of tccc itself is the
 // trailing CCC shifted left 2 bits. The two least-significant bits of tccc
 // are the number of trailing non-starters.
 
 const (
 	qcInfoMask      = 0x3F // to clear all but the relevant bits in a qcInfo
 	headerLenMask   = 0x3F // extract the length value from the header byte
 	headerFlagsMask = 0xC0 // extract the qcInfo bits from the header byte
 )
 
 // Properties provides access to normalization properties of a rune.
 type Properties struct {
 	pos   uint8  // start position in reorderBuffer; used in composition.go
 	size  uint8  // length of UTF-8 encoding of this rune
 	ccc   uint8  // leading canonical combining class (ccc if not decomposition)
 	tccc  uint8  // trailing canonical combining class (ccc if not decomposition)
 	nLead uint8  // number of leading non-starters.
 	flags qcInfo // quick check flags
 	index uint16
 }
 
 // functions dispatchable per form
 type lookupFunc func(b input, i int) Properties
 
 // formInfo holds Form-specific functions and tables.
 type formInfo struct {
 	form                     Form
 	composing, compatibility bool // form type
 	info                     lookupFunc
 	nextMain                 iterFunc
 }
 
 var formTable = []*formInfo{{
 	form:          NFC,
 	composing:     true,
 	compatibility: false,
 	info:          lookupInfoNFC,
 	nextMain:      nextComposed,
 }, {
 	form:          NFD,
 	composing:     false,
 	compatibility: false,
 	info:          lookupInfoNFC,
 	nextMain:      nextDecomposed,
 }, {
 	form:          NFKC,
 	composing:     true,
 	compatibility: true,
 	info:          lookupInfoNFKC,
 	nextMain:      nextComposed,
 }, {
 	form:          NFKD,
 	composing:     false,
 	compatibility: true,
 	info:          lookupInfoNFKC,
 	nextMain:      nextDecomposed,
 }}
 
 // We do not distinguish between boundaries for NFC, NFD, etc. to avoid
 // unexpected behavior for the user.  For example, in NFD, there is a boundary
 // after 'a'.  However, 'a' might combine with modifiers, so from the application's
 // perspective it is not a good boundary. We will therefore always use the
 // boundaries for the combining variants.
 
 // BoundaryBefore returns true if this rune starts a new segment and
 // cannot combine with any rune on the left.
 func (p Properties) BoundaryBefore() bool {
 	if p.ccc == 0 && !p.combinesBackward() {
 		return true
 	}
 	// We assume that the CCC of the first character in a decomposition
 	// is always non-zero if different from info.ccc and that we can return
 	// false at this point. This is verified by maketables.
 	return false
 }
 
 // BoundaryAfter returns true if runes cannot combine with or otherwise
 // interact with this or previous runes.
 func (p Properties) BoundaryAfter() bool {
 	// TODO: loosen these conditions.
 	return p.isInert()
 }
 
 // We pack quick check data in 4 bits:
 //
 //	5:    Combines forward  (0 == false, 1 == true)
 //	4..3: NFC_QC Yes(00), No (10), or Maybe (11)
 //	2:    NFD_QC Yes (0) or No (1). No also means there is a decomposition.
 //	1..0: Number of trailing non-starters.
 //
 // When all 4 bits are zero, the character is inert, meaning it is never
 // influenced by normalization.
 type qcInfo uint8
 
 func (p Properties) isYesC() bool { return p.flags&0x10 == 0 }
 func (p Properties) isYesD() bool { return p.flags&0x4 == 0 }
 
 func (p Properties) combinesForward() bool  { return p.flags&0x20 != 0 }
 func (p Properties) combinesBackward() bool { return p.flags&0x8 != 0 } // == isMaybe
 func (p Properties) hasDecomposition() bool { return p.flags&0x4 != 0 } // == isNoD
 
 func (p Properties) isInert() bool {
 	return p.flags&qcInfoMask == 0 && p.ccc == 0
 }
 
 func (p Properties) multiSegment() bool {
 	return p.index >= firstMulti && p.index < endMulti
 }
 
 func (p Properties) nLeadingNonStarters() uint8 {
 	return p.nLead
 }
 
 func (p Properties) nTrailingNonStarters() uint8 {
 	return uint8(p.flags & 0x03)
 }
 
 // Decomposition returns the decomposition for the underlying rune
 // or nil if there is none.
 func (p Properties) Decomposition() []byte {
 	// TODO: create the decomposition for Hangul?
 	if p.index == 0 {
 		return nil
 	}
 	i := p.index
 	n := decomps[i] & headerLenMask
 	i++
 	return decomps[i : i+uint16(n)]
 }
 
 // Size returns the length of UTF-8 encoding of the rune.
 func (p Properties) Size() int {
 	return int(p.size)
 }
 
 // CCC returns the canonical combining class of the underlying rune.
 func (p Properties) CCC() uint8 {
 	if p.index >= firstCCCZeroExcept {
 		return 0
 	}
 	return ccc[p.ccc]
 }
 
 // LeadCCC returns the CCC of the first rune in the decomposition.
 // If there is no decomposition, LeadCCC equals CCC.
 func (p Properties) LeadCCC() uint8 {
 	return ccc[p.ccc]
 }
 
 // TrailCCC returns the CCC of the last rune in the decomposition.
 // If there is no decomposition, TrailCCC equals CCC.
 func (p Properties) TrailCCC() uint8 {
 	return ccc[p.tccc]
 }
 
 func buildRecompMap() {
 	recompMap = make(map[uint32]rune, len(recompMapPacked)/8)
 	var buf [8]byte
 	for i := 0; i < len(recompMapPacked); i += 8 {
 		copy(buf[:], recompMapPacked[i:i+8])
 		key := binary.BigEndian.Uint32(buf[:4])
 		val := binary.BigEndian.Uint32(buf[4:])
 		recompMap[key] = rune(val)
 	}
 }
 
 // Recomposition
 // We use 32-bit keys instead of 64-bit for the two codepoint keys.
 // This clips off the bits of three entries, but we know this will not
 // result in a collision. In the unlikely event that changes to
 // UnicodeData.txt introduce collisions, the compiler will catch it.
 // Note that the recomposition map for NFC and NFKC are identical.
 
 // combine returns the combined rune or 0 if it doesn't exist.
 //
 // The caller is responsible for calling
 // recompMapOnce.Do(buildRecompMap) sometime before this is called.
 func combine(a, b rune) rune {
 	key := uint32(uint16(a))<<16 + uint32(uint16(b))
 	if recompMap == nil {
 		panic("caller error") // see func comment
 	}
 	return recompMap[key]
 }
 
 func lookupInfoNFC(b input, i int) Properties {
 	v, sz := b.charinfoNFC(i)
 	return compInfo(v, sz)
 }
 
 func lookupInfoNFKC(b input, i int) Properties {
 	v, sz := b.charinfoNFKC(i)
 	return compInfo(v, sz)
 }
 
 // Properties returns properties for the first rune in s.
 func (f Form) Properties(s []byte) Properties {
 	if f == NFC || f == NFD {
 		return compInfo(nfcData.lookup(s))
 	}
 	return compInfo(nfkcData.lookup(s))
 }
 
 // PropertiesString returns properties for the first rune in s.
 func (f Form) PropertiesString(s string) Properties {
 	if f == NFC || f == NFD {
 		return compInfo(nfcData.lookupString(s))
 	}
 	return compInfo(nfkcData.lookupString(s))
 }
 
 // compInfo converts the information contained in v and sz
 // to a Properties.  See the comment at the top of the file
 // for more information on the format.
 func compInfo(v uint16, sz int) Properties {
 	if v == 0 {
 		return Properties{size: uint8(sz)}
 	} else if v >= 0x8000 {
 		p := Properties{
 			size:  uint8(sz),
 			ccc:   uint8(v),
 			tccc:  uint8(v),
 			flags: qcInfo(v >> 8),
 		}
 		if p.ccc > 0 || p.combinesBackward() {
 			p.nLead = uint8(p.flags & 0x3)
 		}
 		return p
 	}
 	// has decomposition
 	h := decomps[v]
 	f := (qcInfo(h&headerFlagsMask) >> 2) | 0x4
 	p := Properties{size: uint8(sz), flags: f, index: v}
 	if v >= firstCCC {
 		v += uint16(h&headerLenMask) + 1
 		c := decomps[v]
 		p.tccc = c >> 2
 		p.flags |= qcInfo(c & 0x3)
 		if v >= firstLeadingCCC {
 			p.nLead = c & 0x3
 			if v >= firstStarterWithNLead {
 				// We were tricked. Remove the decomposition.
 				p.flags &= 0x03
 				p.index = 0
 				return p
 			}
 			p.ccc = decomps[v+1]
 		}
 	}
 	return p
 }