kakoune

Fork of kakoune, an acceptable text editor
git clone git://git.laack.co/kakoune.git
Log | Files | Refs

utf8.hh (7706B)


      1 #ifndef utf8_hh_INCLUDED
      2 #define utf8_hh_INCLUDED
      3 
      4 #include "assert.hh"
      5 #include "unicode.hh"
      6 #include "units.hh"
      7 
      8 namespace Kakoune
      9 {
     10 
     11 namespace utf8
     12 {
     13 
     14 template<typename Iterator>
     15 [[gnu::always_inline]]
     16 inline char read(Iterator& it) noexcept { char c = *it; ++it; return c; }
     17 
     18 // return true if it points to the first byte of a (either single or
     19 // multibyte) character
     20 [[gnu::always_inline]]
     21 inline bool is_character_start(char c) noexcept
     22 {
     23     return (c & 0xC0) != 0x80;
     24 }
     25 
     26 namespace InvalidPolicy
     27 {
     28 
     29 struct Assert
     30 {
     31     Codepoint operator()(Codepoint cp) const { kak_assert(false); return cp; }
     32 };
     33 
     34 struct Pass
     35 {
     36     Codepoint operator()(Codepoint cp) const noexcept { return cp; }
     37 };
     38 
     39 }
     40 
     41 template<typename InvalidPolicy = utf8::InvalidPolicy::Pass,
     42          typename Iterator, typename Sentinel>
     43 [[gnu::noinline]]
     44 Codepoint read_codepoint_multibyte(Iterator& it, const Sentinel& end, char byte)
     45     noexcept(noexcept(InvalidPolicy{}(0)))
     46 {
     47     if (it == end)
     48         return InvalidPolicy{}(byte);
     49 
     50     if ((byte & 0xE0) == 0xC0) // 110xxxxx
     51         return ((byte & 0x1F) << 6) | (read(it) & 0x3F);
     52 
     53     if ((byte & 0xF0) == 0xE0) // 1110xxxx
     54     {
     55         Codepoint cp = ((byte & 0x0F) << 12) | ((read(it) & 0x3F) << 6);
     56         if (it == end)
     57             return InvalidPolicy{}(cp);
     58         return cp | (read(it) & 0x3F);
     59     }
     60 
     61     if ((byte & 0xF8) == 0xF0) // 11110xxx
     62     {
     63         Codepoint cp = ((byte & 0x0F) << 18) | ((read(it) & 0x3F) << 12);
     64         if (it == end)
     65             return InvalidPolicy{}(cp);
     66         cp |= (read(it) & 0x3F) << 6;
     67         if (it == end)
     68             return InvalidPolicy{}(cp);
     69         return cp | (read(it) & 0x3F);
     70     }
     71     return InvalidPolicy{}(byte);
     72 }
     73 
     74 // returns the codepoint of the character whose first byte
     75 // is pointed by it
     76 template<typename InvalidPolicy = utf8::InvalidPolicy::Pass,
     77          typename Iterator, typename Sentinel>
     78 Codepoint read_codepoint(Iterator& it, const Sentinel& end)
     79     noexcept(noexcept(InvalidPolicy{}(0)))
     80 {
     81     if (it == end)
     82         return InvalidPolicy{}(-1);
     83     unsigned char byte = read(it);
     84     if ((byte & 0x80) == 0) [[likely]] // 0xxxxxxx
     85         return byte;
     86     return read_codepoint_multibyte(it, end, byte);
     87 }
     88 
     89 template<typename InvalidPolicy = utf8::InvalidPolicy::Pass,
     90          typename Iterator, typename Sentinel>
     91 Codepoint codepoint(Iterator it, const Sentinel& end)
     92     noexcept(noexcept(read_codepoint<InvalidPolicy>(it, end)))
     93 {
     94     return read_codepoint<InvalidPolicy>(it, end);
     95 }
     96 
     97 template<typename InvalidPolicy = utf8::InvalidPolicy::Pass>
     98 ByteCount codepoint_size(char byte)
     99     noexcept(noexcept(InvalidPolicy{}(0)))
    100 {
    101     if ((byte & 0x80) == 0) // 0xxxxxxx
    102         return 1;
    103     else if ((byte & 0xE0) == 0xC0) // 110xxxxx
    104         return 2;
    105     else if ((byte & 0xF0) == 0xE0) // 1110xxxx
    106         return 3;
    107     else if ((byte & 0xF8) == 0xF0) // 11110xxx
    108         return 4;
    109     else
    110     {
    111         InvalidPolicy{}(byte);
    112         return 1;
    113     }
    114 }
    115 
    116 template<typename InvalidPolicy = utf8::InvalidPolicy::Pass>
    117 ByteCount codepoint_size(Codepoint cp)
    118     noexcept(noexcept(InvalidPolicy{}(0)))
    119 {
    120     if (cp <= 0x7F)
    121         return 1;
    122     else if (cp <= 0x7FF)
    123         return 2;
    124     else if (cp <= 0xFFFF)
    125         return 3;
    126     else if (cp <= 0x10FFFF)
    127         return 4;
    128     else
    129     {
    130         InvalidPolicy{}(cp);
    131         return 0;
    132     }
    133 }
    134 
    135 template<typename Iterator, typename Sentinel>
    136 void to_next(Iterator& it, const Sentinel& end) noexcept
    137 {
    138     if (it != end)
    139         ++it;
    140     while (it != end and not is_character_start(*it))
    141         ++it;
    142 }
    143 
    144 // returns an iterator to next character first byte
    145 template<typename Iterator, typename Sentinel>
    146 Iterator next(Iterator it, const Sentinel& end) noexcept
    147 {
    148     to_next(it, end);
    149     return it;
    150 }
    151 
    152 // returns it's parameter if it points to a character first byte,
    153 // or else returns next character first byte
    154 template<typename Iterator, typename Sentinel>
    155 Iterator finish(Iterator it, const Sentinel& end) noexcept
    156 {
    157     while (it != end and (*(it) & 0xC0) == 0x80)
    158         ++it;
    159     return it;
    160 }
    161 
    162 template<typename Iterator, typename Sentinel>
    163 void to_previous(Iterator& it, const Sentinel& begin) noexcept
    164 {
    165     if (it != begin)
    166         --it;
    167     while (it != begin and not is_character_start(*it))
    168         --it;
    169 }
    170 // returns an iterator to the previous character first byte
    171 template<typename Iterator, typename Sentinel>
    172 Iterator previous(Iterator it, const Sentinel& begin) noexcept
    173 {
    174     to_previous(it, begin);
    175     return it;
    176 }
    177 
    178 // returns an iterator pointing to the first byte of the
    179 // dth character after (or before if d < 0) the character
    180 // pointed by it
    181 template<typename Iterator, typename Sentinel>
    182 Iterator advance(Iterator it, const Sentinel& end, CharCount d) noexcept
    183 {
    184     if (it == end)
    185         return it;
    186 
    187     if (d < 0)
    188     {
    189         while (it != end and d++ != 0)
    190             to_previous(it, end);
    191     }
    192     else if (d > 0)
    193     {
    194         while (it != end and d-- != 0)
    195             to_next(it, end);
    196     }
    197     return it;
    198 }
    199 
    200 // returns an iterator pointing to the first byte of the
    201 // character at the dth column after (or before if d < 0)
    202 // the character pointed by it
    203 template<typename Iterator, typename Sentinel>
    204 Iterator advance(Iterator it, const Sentinel& end, ColumnCount d) noexcept
    205 {
    206     if (it == end)
    207         return it;
    208 
    209     if (d < 0)
    210     {
    211         while (it != end and d < 0)
    212         {
    213             auto cur = it;
    214             to_previous(it, end);
    215             d += codepoint_width(codepoint(it, cur));
    216         }
    217     }
    218     else if (d > 0)
    219     {
    220         auto begin = it;
    221         while (it != end and d > 0)
    222         {
    223             d -= codepoint_width(read_codepoint(it, end));
    224             if (it != end and d < 0)
    225                 to_previous(it, begin);
    226         }
    227     }
    228     return it;
    229 }
    230 
    231 // returns the character count between begin and end
    232 template<typename Iterator, typename Sentinel>
    233 CharCount distance(Iterator begin, const Sentinel& end) noexcept
    234 {
    235     CharCount dist = 0;
    236 
    237     while (begin != end)
    238     {
    239         if (is_character_start(read(begin)))
    240             ++dist;
    241     }
    242     return dist;
    243 }
    244 
    245 // returns the column count between begin and end
    246 template<typename Iterator, typename Sentinel>
    247 ColumnCount column_distance(Iterator begin, const Sentinel& end) noexcept
    248 {
    249     ColumnCount dist = 0;
    250 
    251     while (begin != end)
    252         dist += codepoint_width(read_codepoint(begin, end));
    253     return dist;
    254 }
    255 
    256 // returns an iterator to the first byte of the character it is into
    257 template<typename Iterator, typename Sentinel>
    258 Iterator character_start(Iterator it, const Sentinel& begin) noexcept
    259 {
    260     while (it != begin and not is_character_start(*it))
    261         --it;
    262     return it;
    263 }
    264 
    265 template<typename InvalidPolicy = utf8::InvalidPolicy::Pass,
    266          typename Iterator, typename Sentinel>
    267 Codepoint prev_codepoint(Iterator it, const Sentinel& begin) noexcept
    268 {
    269     if (it <= begin)
    270         return InvalidPolicy{}(-1);
    271     return codepoint<InvalidPolicy>(character_start(it -1, begin), it);
    272 }
    273 
    274 
    275 template<typename OutputIterator, typename InvalidPolicy = utf8::InvalidPolicy::Pass>
    276 void dump(OutputIterator&& it, Codepoint cp)
    277 {
    278     if (cp <= 0x7F)
    279         *it++ = cp;
    280     else if (cp <= 0x7FF)
    281     {
    282         *it++ = 0xC0 | (cp >> 6);
    283         *it++ = 0x80 | (cp & 0x3F);
    284     }
    285     else if (cp <= 0xFFFF)
    286     {
    287         *it++ = 0xE0 | (cp >> 12);
    288         *it++ = 0x80 | ((cp >> 6) & 0x3F);
    289         *it++ = 0x80 | (cp & 0x3F);
    290     }
    291     else if (cp <= 0x10FFFF)
    292     {
    293         *it++ = 0xF0 | (cp >> 18);
    294         *it++ = 0x80 | ((cp >> 12) & 0x3F);
    295         *it++ = 0x80 | ((cp >> 6)  & 0x3F);
    296         *it++ = 0x80 | (cp & 0x3F);
    297     }
    298     else
    299         InvalidPolicy{}(cp);
    300 }
    301 
    302 }
    303 
    304 }
    305 
    306 #endif // utf8_hh_INCLUDED