sdk/tools/unicode/utf8.c

   1 /*
   2  * UTF-8 support routines
   3  *
   4  * Copyright 2000 Alexandre Julliard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
  19  */
  20
  21 #include <string.h>
  22
  23 #include "wine/unicode.h"
  24
  25 extern WCHAR wine_compose( const WCHAR *str );
  26
  27 /* number of following bytes in sequence based on first byte value (for bytes above 0x7f) */
  28 static const char utf8_length[128] =
  29 {
  30     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 0x80-0x8f */
  31     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 0x90-0x9f */
  32     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 0xa0-0xaf */
  33     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 0xb0-0xbf */
  34     0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* 0xc0-0xcf */
  35     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* 0xd0-0xdf */
  36     2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, /* 0xe0-0xef */
  37     3,3,3,3,3,0,0,0,0,0,0,0,0,0,0,0  /* 0xf0-0xff */
  38 };
  39
  40 /* first byte mask depending on UTF-8 sequence length */
  41 static const unsigned char utf8_mask[4] = { 0x7f, 0x1f, 0x0f, 0x07 };
  42
  43 /* minimum Unicode value depending on UTF-8 sequence length */
  44 static const unsigned int utf8_minval[4] = { 0x0, 0x80, 0x800, 0x10000 };
  45
  46
  47 /* get the next char value taking surrogates into account */
  48 static inline unsigned int get_surrogate_value( const WCHAR *src, unsigned int srclen )
  49 {
  50     if (src[0] >= 0xd800 && src[0] <= 0xdfff)  /* surrogate pair */
  51     {
  52         if (src[0] > 0xdbff || /* invalid high surrogate */
  53             srclen <= 1 ||     /* missing low surrogate */
  54             src[1] < 0xdc00 || src[1] > 0xdfff) /* invalid low surrogate */
  55             return 0;
  56         return 0x10000 + ((src[0] & 0x3ff) << 10) + (src[1] & 0x3ff);
  57     }
  58     return src[0];
  59 }
  60
  61 /* query necessary dst length for src string */
  62 static inline int get_length_wcs_utf8( int flags, const WCHAR *src, unsigned int srclen )
  63 {
  64     int len;
  65     unsigned int val;
  66
  67     for (len = 0; srclen; srclen--, src++)
  68     {
  69         if (*src < 0x80)  /* 0x00-0x7f: 1 byte */
  70         {
  71             len++;
  72             continue;
  73         }
  74         if (*src < 0x800)  /* 0x80-0x7ff: 2 bytes */
  75         {
  76             len += 2;
  77             continue;
  78         }
  79         if (!(val = get_surrogate_value( src, srclen )))
  80         {
  81             if (flags & WC_ERR_INVALID_CHARS) return -2;
  82             continue;
  83         }
  84         if (val < 0x10000)  /* 0x800-0xffff: 3 bytes */
  85             len += 3;
  86         else   /* 0x10000-0x10ffff: 4 bytes */
  87         {
  88             len += 4;
  89             src++;
  90             srclen--;
  91         }
  92     }
  93     return len;
  94 }
  95
  96 /* wide char to UTF-8 string conversion */
  97 /* return -1 on dst buffer overflow, -2 on invalid input char */
  98 int wine_utf8_wcstombs( int flags, const WCHAR *src, int srclen, char *dst, int dstlen )
  99 {
 100     int len;
 101
 102     if (!dstlen) return get_length_wcs_utf8( flags, src, srclen );
 103
 104     for (len = dstlen; srclen; srclen--, src++)
 105     {
 106         WCHAR ch = *src;
 107         unsigned int val;
 108
 109         if (ch < 0x80)  /* 0x00-0x7f: 1 byte */
 110         {
 111             if (!len--) return -1;  /* overflow */
 112             *dst++ = ch;
 113             continue;
 114         }
 115
 116         if (ch < 0x800)  /* 0x80-0x7ff: 2 bytes */
 117         {
 118             if ((len -= 2) < 0) return -1;  /* overflow */
 119             dst[1] = 0x80 | (ch & 0x3f);
 120             ch >>= 6;
 121             dst[0] = 0xc0 | ch;
 122             dst += 2;
 123             continue;
 124         }
 125
 126         if (!(val = get_surrogate_value( src, srclen )))
 127         {
 128             if (flags & WC_ERR_INVALID_CHARS) return -2;
 129             continue;
 130         }
 131
 132         if (val < 0x10000)  /* 0x800-0xffff: 3 bytes */
 133         {
 134             if ((len -= 3) < 0) return -1;  /* overflow */
 135             dst[2] = 0x80 | (val & 0x3f);
 136             val >>= 6;
 137             dst[1] = 0x80 | (val & 0x3f);
 138             val >>= 6;
 139             dst[0] = 0xe0 | val;
 140             dst += 3;
 141         }
 142         else   /* 0x10000-0x10ffff: 4 bytes */
 143         {
 144             if ((len -= 4) < 0) return -1;  /* overflow */
 145             dst[3] = 0x80 | (val & 0x3f);
 146             val >>= 6;
 147             dst[2] = 0x80 | (val & 0x3f);
 148             val >>= 6;
 149             dst[1] = 0x80 | (val & 0x3f);
 150             val >>= 6;
 151             dst[0] = 0xf0 | val;
 152             dst += 4;
 153             src++;
 154             srclen--;
 155         }
 156     }
 157     return dstlen - len;
 158 }
 159
 160 /* helper for the various utf8 mbstowcs functions */
 161 static inline unsigned int decode_utf8_char( unsigned char ch, const char **str, const char *strend )
 162 {
 163     unsigned int len = utf8_length[ch-0x80];
 164     unsigned int res = ch & utf8_mask[len];
 165     const char *end = *str + len;
 166
 167     if (end > strend) return ~0;
 168     switch(len)
 169     {
 170     case 3:
 171         if ((ch = end[-3] ^ 0x80) >= 0x40) break;
 172         res = (res << 6) | ch;
 173         (*str)++;
 174     case 2:
 175         if ((ch = end[-2] ^ 0x80) >= 0x40) break;
 176         res = (res << 6) | ch;
 177         (*str)++;
 178     case 1:
 179         if ((ch = end[-1] ^ 0x80) >= 0x40) break;
 180         res = (res << 6) | ch;
 181         (*str)++;
 182         if (res < utf8_minval[len]) break;
 183         return res;
 184     }
 185     return ~0;
 186 }
 187
 188 /* query necessary dst length for src string with composition */
 189 static inline int get_length_mbs_utf8_compose( int flags, const char *src, int srclen )
 190 {
 191     int ret = 0;
 192     unsigned int res;
 193     WCHAR composed[2];
 194     const char *srcend = src + srclen;
 195
 196     composed[0] = 0;
 197     while (src < srcend)
 198     {
 199         unsigned char ch = *src++;
 200         if (ch < 0x80)  /* special fast case for 7-bit ASCII */
 201         {
 202             composed[0] = ch;
 203             ret++;
 204             continue;
 205         }
 206         if ((res = decode_utf8_char( ch, &src, srcend )) <= 0xffff)
 207         {
 208             if (composed[0])
 209             {
 210                 composed[1] = res;
 211                 if ((composed[0] = wine_compose( composed ))) continue;
 212             }
 213             composed[0] = res;
 214             ret++;
 215         }
 216         else if (res <= 0x10ffff)
 217         {
 218             ret += 2;
 219             composed[0] = 0;  /* no composition for surrogates */
 220         }
 221         else if (flags & MB_ERR_INVALID_CHARS) return -2;  /* bad char */
 222         /* otherwise ignore it */
 223     }
 224     return ret;
 225 }
 226
 227 /* UTF-8 to wide char string conversion with composition */
 228 /* return -1 on dst buffer overflow, -2 on invalid input char */
 229 static int utf8_mbstowcs_compose( int flags, const char *src, int srclen, WCHAR *dst, int dstlen )
 230 {
 231     unsigned int res;
 232     const char *srcend = src + srclen;
 233     WCHAR composed[2];
 234     WCHAR *dstend = dst + dstlen;
 235
 236     if (!dstlen) return get_length_mbs_utf8_compose( flags, src, srclen );
 237
 238     composed[0] = 0;
 239     while (src < srcend)
 240     {
 241         unsigned char ch = *src++;
 242         if (ch < 0x80)  /* special fast case for 7-bit ASCII */
 243         {
 244             if (dst >= dstend) return -1;  /* overflow */
 245             *dst++ = composed[0] = ch;
 246             continue;
 247         }
 248         if ((res = decode_utf8_char( ch, &src, srcend )) <= 0xffff)
 249         {
 250             if (composed[0])
 251             {
 252                 composed[1] = res;
 253                 if ((composed[0] = wine_compose( composed )))
 254                 {
 255                     dst[-1] = composed[0];
 256                     continue;
 257                 }
 258             }
 259             if (dst >= dstend) return -1;  /* overflow */
 260             *dst++ = composed[0] = res;
 261         }
 262         else if (res <= 0x10ffff) /* we need surrogates */
 263         {
 264             if (dst >= dstend - 1) return -1;  /* overflow */
 265             res -= 0x10000;
 266             *dst++ = 0xd800 | (res >> 10);
 267             *dst++ = 0xdc00 | (res & 0x3ff);
 268             composed[0] = 0;  /* no composition for surrogates */
 269         }
 270         else if (flags & MB_ERR_INVALID_CHARS) return -2;  /* bad char */
 271         /* otherwise ignore it */
 272     }
 273     return dstlen - (dstend - dst);
 274 }
 275
 276 /* query necessary dst length for src string */
 277 static inline int get_length_mbs_utf8( int flags, const char *src, int srclen )
 278 {
 279     int ret = 0;
 280     unsigned int res;
 281     const char *srcend = src + srclen;
 282
 283     while (src < srcend)
 284     {
 285         unsigned char ch = *src++;
 286         if (ch < 0x80)  /* special fast case for 7-bit ASCII */
 287         {
 288             ret++;
 289             continue;
 290         }
 291         if ((res = decode_utf8_char( ch, &src, srcend )) <= 0x10ffff)
 292         {
 293             if (res > 0xffff) ret++;
 294             ret++;
 295         }
 296         else if (flags & MB_ERR_INVALID_CHARS) return -2;  /* bad char */
 297         /* otherwise ignore it */
 298     }
 299     return ret;
 300 }
 301
 302 /* UTF-8 to wide char string conversion */
 303 /* return -1 on dst buffer overflow, -2 on invalid input char */
 304 int wine_utf8_mbstowcs( int flags, const char *src, int srclen, WCHAR *dst, int dstlen )
 305 {
 306     unsigned int res;
 307     const char *srcend = src + srclen;
 308     WCHAR *dstend = dst + dstlen;
 309
 310     if (flags & MB_COMPOSITE) return utf8_mbstowcs_compose( flags, src, srclen, dst, dstlen );
 311
 312     if (!dstlen) return get_length_mbs_utf8( flags, src, srclen );
 313
 314     while ((dst < dstend) && (src < srcend))
 315     {
 316         unsigned char ch = *src++;
 317         if (ch < 0x80)  /* special fast case for 7-bit ASCII */
 318         {
 319             *dst++ = ch;
 320             continue;
 321         }
 322         if ((res = decode_utf8_char( ch, &src, srcend )) <= 0xffff)
 323         {
 324             *dst++ = res;
 325         }
 326         else if (res <= 0x10ffff)  /* we need surrogates */
 327         {
 328             if (dst == dstend - 1) return -1;  /* overflow */
 329             res -= 0x10000;
 330             *dst++ = 0xd800 | (res >> 10);
 331             *dst++ = 0xdc00 | (res & 0x3ff);
 332         }
 333         else if (flags & MB_ERR_INVALID_CHARS) return -2;  /* bad char */
 334         /* otherwise ignore it */
 335     }
 336     if (src < srcend) return -1;  /* overflow */
 337     return dstlen - (dstend - dst);
 338 }