utf8.hh (7706B)
1 #ifndef utf8_hh_INCLUDED 2 #define utf8_hh_INCLUDED 3 4 #include "assert.hh" 5 #include "unicode.hh" 6 #include "units.hh" 7 8 namespace Kakoune 9 { 10 11 namespace utf8 12 { 13 14 template<typename Iterator> 15 [[gnu::always_inline]] 16 inline char read(Iterator& it) noexcept { char c = *it; ++it; return c; } 17 18 // return true if it points to the first byte of a (either single or 19 // multibyte) character 20 [[gnu::always_inline]] 21 inline bool is_character_start(char c) noexcept 22 { 23 return (c & 0xC0) != 0x80; 24 } 25 26 namespace InvalidPolicy 27 { 28 29 struct Assert 30 { 31 Codepoint operator()(Codepoint cp) const { kak_assert(false); return cp; } 32 }; 33 34 struct Pass 35 { 36 Codepoint operator()(Codepoint cp) const noexcept { return cp; } 37 }; 38 39 } 40 41 template<typename InvalidPolicy = utf8::InvalidPolicy::Pass, 42 typename Iterator, typename Sentinel> 43 [[gnu::noinline]] 44 Codepoint read_codepoint_multibyte(Iterator& it, const Sentinel& end, char byte) 45 noexcept(noexcept(InvalidPolicy{}(0))) 46 { 47 if (it == end) 48 return InvalidPolicy{}(byte); 49 50 if ((byte & 0xE0) == 0xC0) // 110xxxxx 51 return ((byte & 0x1F) << 6) | (read(it) & 0x3F); 52 53 if ((byte & 0xF0) == 0xE0) // 1110xxxx 54 { 55 Codepoint cp = ((byte & 0x0F) << 12) | ((read(it) & 0x3F) << 6); 56 if (it == end) 57 return InvalidPolicy{}(cp); 58 return cp | (read(it) & 0x3F); 59 } 60 61 if ((byte & 0xF8) == 0xF0) // 11110xxx 62 { 63 Codepoint cp = ((byte & 0x0F) << 18) | ((read(it) & 0x3F) << 12); 64 if (it == end) 65 return InvalidPolicy{}(cp); 66 cp |= (read(it) & 0x3F) << 6; 67 if (it == end) 68 return InvalidPolicy{}(cp); 69 return cp | (read(it) & 0x3F); 70 } 71 return InvalidPolicy{}(byte); 72 } 73 74 // returns the codepoint of the character whose first byte 75 // is pointed by it 76 template<typename InvalidPolicy = utf8::InvalidPolicy::Pass, 77 typename Iterator, typename Sentinel> 78 Codepoint read_codepoint(Iterator& it, const Sentinel& end) 79 noexcept(noexcept(InvalidPolicy{}(0))) 80 { 81 if (it == end) 82 return InvalidPolicy{}(-1); 83 unsigned char byte = read(it); 84 if ((byte & 0x80) == 0) [[likely]] // 0xxxxxxx 85 return byte; 86 return read_codepoint_multibyte(it, end, byte); 87 } 88 89 template<typename InvalidPolicy = utf8::InvalidPolicy::Pass, 90 typename Iterator, typename Sentinel> 91 Codepoint codepoint(Iterator it, const Sentinel& end) 92 noexcept(noexcept(read_codepoint<InvalidPolicy>(it, end))) 93 { 94 return read_codepoint<InvalidPolicy>(it, end); 95 } 96 97 template<typename InvalidPolicy = utf8::InvalidPolicy::Pass> 98 ByteCount codepoint_size(char byte) 99 noexcept(noexcept(InvalidPolicy{}(0))) 100 { 101 if ((byte & 0x80) == 0) // 0xxxxxxx 102 return 1; 103 else if ((byte & 0xE0) == 0xC0) // 110xxxxx 104 return 2; 105 else if ((byte & 0xF0) == 0xE0) // 1110xxxx 106 return 3; 107 else if ((byte & 0xF8) == 0xF0) // 11110xxx 108 return 4; 109 else 110 { 111 InvalidPolicy{}(byte); 112 return 1; 113 } 114 } 115 116 template<typename InvalidPolicy = utf8::InvalidPolicy::Pass> 117 ByteCount codepoint_size(Codepoint cp) 118 noexcept(noexcept(InvalidPolicy{}(0))) 119 { 120 if (cp <= 0x7F) 121 return 1; 122 else if (cp <= 0x7FF) 123 return 2; 124 else if (cp <= 0xFFFF) 125 return 3; 126 else if (cp <= 0x10FFFF) 127 return 4; 128 else 129 { 130 InvalidPolicy{}(cp); 131 return 0; 132 } 133 } 134 135 template<typename Iterator, typename Sentinel> 136 void to_next(Iterator& it, const Sentinel& end) noexcept 137 { 138 if (it != end) 139 ++it; 140 while (it != end and not is_character_start(*it)) 141 ++it; 142 } 143 144 // returns an iterator to next character first byte 145 template<typename Iterator, typename Sentinel> 146 Iterator next(Iterator it, const Sentinel& end) noexcept 147 { 148 to_next(it, end); 149 return it; 150 } 151 152 // returns it's parameter if it points to a character first byte, 153 // or else returns next character first byte 154 template<typename Iterator, typename Sentinel> 155 Iterator finish(Iterator it, const Sentinel& end) noexcept 156 { 157 while (it != end and (*(it) & 0xC0) == 0x80) 158 ++it; 159 return it; 160 } 161 162 template<typename Iterator, typename Sentinel> 163 void to_previous(Iterator& it, const Sentinel& begin) noexcept 164 { 165 if (it != begin) 166 --it; 167 while (it != begin and not is_character_start(*it)) 168 --it; 169 } 170 // returns an iterator to the previous character first byte 171 template<typename Iterator, typename Sentinel> 172 Iterator previous(Iterator it, const Sentinel& begin) noexcept 173 { 174 to_previous(it, begin); 175 return it; 176 } 177 178 // returns an iterator pointing to the first byte of the 179 // dth character after (or before if d < 0) the character 180 // pointed by it 181 template<typename Iterator, typename Sentinel> 182 Iterator advance(Iterator it, const Sentinel& end, CharCount d) noexcept 183 { 184 if (it == end) 185 return it; 186 187 if (d < 0) 188 { 189 while (it != end and d++ != 0) 190 to_previous(it, end); 191 } 192 else if (d > 0) 193 { 194 while (it != end and d-- != 0) 195 to_next(it, end); 196 } 197 return it; 198 } 199 200 // returns an iterator pointing to the first byte of the 201 // character at the dth column after (or before if d < 0) 202 // the character pointed by it 203 template<typename Iterator, typename Sentinel> 204 Iterator advance(Iterator it, const Sentinel& end, ColumnCount d) noexcept 205 { 206 if (it == end) 207 return it; 208 209 if (d < 0) 210 { 211 while (it != end and d < 0) 212 { 213 auto cur = it; 214 to_previous(it, end); 215 d += codepoint_width(codepoint(it, cur)); 216 } 217 } 218 else if (d > 0) 219 { 220 auto begin = it; 221 while (it != end and d > 0) 222 { 223 d -= codepoint_width(read_codepoint(it, end)); 224 if (it != end and d < 0) 225 to_previous(it, begin); 226 } 227 } 228 return it; 229 } 230 231 // returns the character count between begin and end 232 template<typename Iterator, typename Sentinel> 233 CharCount distance(Iterator begin, const Sentinel& end) noexcept 234 { 235 CharCount dist = 0; 236 237 while (begin != end) 238 { 239 if (is_character_start(read(begin))) 240 ++dist; 241 } 242 return dist; 243 } 244 245 // returns the column count between begin and end 246 template<typename Iterator, typename Sentinel> 247 ColumnCount column_distance(Iterator begin, const Sentinel& end) noexcept 248 { 249 ColumnCount dist = 0; 250 251 while (begin != end) 252 dist += codepoint_width(read_codepoint(begin, end)); 253 return dist; 254 } 255 256 // returns an iterator to the first byte of the character it is into 257 template<typename Iterator, typename Sentinel> 258 Iterator character_start(Iterator it, const Sentinel& begin) noexcept 259 { 260 while (it != begin and not is_character_start(*it)) 261 --it; 262 return it; 263 } 264 265 template<typename InvalidPolicy = utf8::InvalidPolicy::Pass, 266 typename Iterator, typename Sentinel> 267 Codepoint prev_codepoint(Iterator it, const Sentinel& begin) noexcept 268 { 269 if (it <= begin) 270 return InvalidPolicy{}(-1); 271 return codepoint<InvalidPolicy>(character_start(it -1, begin), it); 272 } 273 274 275 template<typename OutputIterator, typename InvalidPolicy = utf8::InvalidPolicy::Pass> 276 void dump(OutputIterator&& it, Codepoint cp) 277 { 278 if (cp <= 0x7F) 279 *it++ = cp; 280 else if (cp <= 0x7FF) 281 { 282 *it++ = 0xC0 | (cp >> 6); 283 *it++ = 0x80 | (cp & 0x3F); 284 } 285 else if (cp <= 0xFFFF) 286 { 287 *it++ = 0xE0 | (cp >> 12); 288 *it++ = 0x80 | ((cp >> 6) & 0x3F); 289 *it++ = 0x80 | (cp & 0x3F); 290 } 291 else if (cp <= 0x10FFFF) 292 { 293 *it++ = 0xF0 | (cp >> 18); 294 *it++ = 0x80 | ((cp >> 12) & 0x3F); 295 *it++ = 0x80 | ((cp >> 6) & 0x3F); 296 *it++ = 0x80 | (cp & 0x3F); 297 } 298 else 299 InvalidPolicy{}(cp); 300 } 301 302 } 303 304 } 305 306 #endif // utf8_hh_INCLUDED