2 * iconv implementation using Win32 API to convert.
4 * This file is placed in the public domain.
7 /* for WC_NO_BEST_FIT_CHARS */
19 #define UNUSED __attribute__((unused))
26 #define GetProcAddressA GetProcAddress
32 # define USE_LIBICONV_DLL
35 #if !defined(DEFAULT_LIBICONV_DLL)
36 # define DEFAULT_LIBICONV_DLL ""
39 #define MB_CHAR_MAX 16
41 #define UNICODE_MODE_BOM_DONE 1
42 #define UNICODE_MODE_SWAPPED 2
44 #define FLAG_USE_BOM 1
45 #define FLAG_TRANSLIT 2 /* //TRANSLIT */
46 #define FLAG_IGNORE 4 /* //IGNORE */
48 typedef unsigned char uchar
;
49 typedef unsigned short ushort
;
50 typedef unsigned int uint
;
52 typedef void* iconv_t
;
54 iconv_t
iconv_open(const char *tocode
, const char *fromcode
);
55 int iconv_close(iconv_t cd
);
56 size_t iconv(iconv_t cd
, const char **inbuf
, size_t *inbytesleft
, char **outbuf
, size_t *outbytesleft
);
58 /* libiconv interface for vim */
61 iconvctl (iconv_t cd
, int request
, void* argument
)
68 typedef struct compat_t compat_t
;
69 typedef struct csconv_t csconv_t
;
70 typedef struct rec_iconv_t rec_iconv_t
;
72 typedef iconv_t (*f_iconv_open
)(const char *tocode
, const char *fromcode
);
73 typedef int (*f_iconv_close
)(iconv_t cd
);
74 typedef size_t (*f_iconv
)(iconv_t cd
, const char **inbuf
, size_t *inbytesleft
, char **outbuf
, size_t *outbytesleft
);
75 typedef int* (*f_errno
)(void);
76 typedef int (*f_mbtowc
)(csconv_t
*cv
, const uchar
*buf
, int bufsize
, ushort
*wbuf
, int *wbufsize
);
77 typedef int (*f_wctomb
)(csconv_t
*cv
, ushort
*wbuf
, int wbufsize
, uchar
*buf
, int bufsize
);
78 typedef int (*f_mblen
)(csconv_t
*cv
, const uchar
*buf
, int bufsize
);
79 typedef int (*f_flush
)(csconv_t
*cv
, uchar
*buf
, int bufsize
);
84 /* unicode mapping for compatibility with other conversion table. */
104 f_iconv_close iconv_close
;
109 #if defined(USE_LIBICONV_DLL)
114 static int win_iconv_open(rec_iconv_t
*cd
, const char *tocode
, const char *fromcode
);
115 static int win_iconv_close(iconv_t cd
);
116 static size_t win_iconv(iconv_t cd
, const char **inbuf
, size_t *inbytesleft
, char **outbuf
, size_t *outbytesleft
);
118 static int load_mlang(void);
119 static int make_csconv(const char *name
, csconv_t
*cv
);
120 static int name_to_codepage(const char *name
);
121 static uint
utf16_to_ucs4(const ushort
*wbuf
);
122 static void ucs4_to_utf16(uint wc
, ushort
*wbuf
, int *wbufsize
);
123 static int mbtowc_flags(int codepage
);
124 static int must_use_null_useddefaultchar(int codepage
);
125 static char *strrstr(const char *str
, const char *token
);
126 static char *xstrndup(const char *s
, size_t n
);
127 static int seterror(int err
);
129 #if defined(USE_LIBICONV_DLL)
130 static int libiconv_iconv_open(rec_iconv_t
*cd
, const char *tocode
, const char *fromcode
);
131 static PVOID
MyImageDirectoryEntryToData(LPVOID Base
, BOOLEAN MappedAsImage
, USHORT DirectoryEntry
, PULONG Size
);
132 static HMODULE
find_imported_module_by_funcname(HMODULE hModule
, const char *funcname
);
134 static HMODULE hwiniconv
;
137 static int sbcs_mblen(csconv_t
*cv
, const uchar
*buf
, int bufsize
);
138 static int dbcs_mblen(csconv_t
*cv
, const uchar
*buf
, int bufsize
);
139 static int mbcs_mblen(csconv_t
*cv
, const uchar
*buf
, int bufsize
);
140 static int utf8_mblen(csconv_t
*cv
, const uchar
*buf
, int bufsize
);
141 static int eucjp_mblen(csconv_t
*cv
, const uchar
*buf
, int bufsize
);
143 static int kernel_mbtowc(csconv_t
*cv
, const uchar
*buf
, int bufsize
, ushort
*wbuf
, int *wbufsize
);
144 static int kernel_wctomb(csconv_t
*cv
, ushort
*wbuf
, int wbufsize
, uchar
*buf
, int bufsize
);
145 static int mlang_mbtowc(csconv_t
*cv
, const uchar
*buf
, int bufsize
, ushort
*wbuf
, int *wbufsize
);
146 static int mlang_wctomb(csconv_t
*cv
, ushort
*wbuf
, int wbufsize
, uchar
*buf
, int bufsize
);
147 static int utf16_mbtowc(csconv_t
*cv
, const uchar
*buf
, int bufsize
, ushort
*wbuf
, int *wbufsize
);
148 static int utf16_wctomb(csconv_t
*cv
, ushort
*wbuf
, int wbufsize
, uchar
*buf
, int bufsize
);
149 static int utf32_mbtowc(csconv_t
*cv
, const uchar
*buf
, int bufsize
, ushort
*wbuf
, int *wbufsize
);
150 static int utf32_wctomb(csconv_t
*cv
, ushort
*wbuf
, int wbufsize
, uchar
*buf
, int bufsize
);
151 static int iso2022jp_mbtowc(csconv_t
*cv
, const uchar
*buf
, int bufsize
, ushort
*wbuf
, int *wbufsize
);
152 static int iso2022jp_wctomb(csconv_t
*cv
, ushort
*wbuf
, int wbufsize
, uchar
*buf
, int bufsize
);
153 static int iso2022jp_flush(csconv_t
*cv
, uchar
*buf
, int bufsize
);
158 } codepage_alias
[] = {
174 {1201, "unicodeFFFE"},
188 #ifndef GLIB_COMPILATION
190 * Default is big endian.
191 * See rfc2781 4.3 Interpreting text labelled as UTF-16.
202 /* Default is little endian, because the platform is */
213 /* copy from libiconv `iconv -l` */
214 /* !IsValidCodePage(367) */
215 {20127, "ANSI_X3.4-1968"},
216 {20127, "ANSI_X3.4-1986"},
221 {20127, "ISO646-US"},
222 {20127, "ISO_646.IRV:1991"},
227 /* !IsValidCodePage(819) */
230 {28591, "ISO-8859-1"},
231 {28591, "ISO-IR-100"},
232 {28591, "ISO8859-1"},
233 {28591, "ISO_8859-1"},
234 {28591, "ISO_8859-1:1987"},
237 {28591, "CSISOLATIN1"},
241 {1250, "WINDOWS-1250"},
245 {1251, "WINDOWS-1251"},
249 {1252, "WINDOWS-1252"},
253 {1253, "WINDOWS-1253"},
257 {1254, "WINDOWS-1254"},
261 {1255, "WINDOWS-1255"},
265 {1256, "WINDOWS-1256"},
268 {1257, "WINBALTRIM"},
269 {1257, "WINDOWS-1257"},
272 {1258, "WINDOWS-1258"},
277 {850, "CSPC850MULTILINGUAL"},
279 /* !IsValidCodePage(862) */
283 {862, "CSPC862LATINHEBREW"},
290 /* !IsValidCodePage(154) */
292 {154, "CYRILLIC-ASIAN"},
297 /* !IsValidCodePage(1133) */
299 {1133, "IBM-CP1133"},
302 {874, "WINDOWS-874"},
304 /* !IsValidCodePage(51932) */
307 {51932, "WINDOWS-51932"},
313 {932, "SHIFFT_JIS-MS"},
318 {932, "WINDOWS-31J"},
319 {932, "WINDOWS-932"},
320 {932, "CSWINDOWS31J"},
323 {50221, "ISO-2022-JP"},
324 {50221, "ISO-2022-JP-MS"},
325 {50221, "ISO2022-JP"},
326 {50221, "ISO2022-JP-MS"},
328 {50221, "WINDOWS-50221"},
333 {936, "WINDOWS-936"},
350 {437, "CSPC8CODEPAGE437"},
356 {775, "CSPC775BALTIC"},
363 /* !IsValidCodePage(853) */
376 /* !IsValidCodePage(858) */
410 /* !IsValidCodePage(1152) */
414 * Code Page Identifiers
415 * http://msdn2.microsoft.com/en-us/library/ms776446.aspx
417 {37, "IBM037"}, /* IBM EBCDIC US-Canada */
418 {437, "IBM437"}, /* OEM United States */
419 {500, "IBM500"}, /* IBM EBCDIC International */
420 {708, "ASMO-708"}, /* Arabic (ASMO 708) */
421 /* 709 Arabic (ASMO-449+, BCON V4) */
422 /* 710 Arabic - Transparent Arabic */
423 {720, "DOS-720"}, /* Arabic (Transparent ASMO); Arabic (DOS) */
424 {737, "ibm737"}, /* OEM Greek (formerly 437G); Greek (DOS) */
425 {775, "ibm775"}, /* OEM Baltic; Baltic (DOS) */
426 {850, "ibm850"}, /* OEM Multilingual Latin 1; Western European (DOS) */
427 {852, "ibm852"}, /* OEM Latin 2; Central European (DOS) */
428 {855, "IBM855"}, /* OEM Cyrillic (primarily Russian) */
429 {857, "ibm857"}, /* OEM Turkish; Turkish (DOS) */
430 {858, "IBM00858"}, /* OEM Multilingual Latin 1 + Euro symbol */
431 {860, "IBM860"}, /* OEM Portuguese; Portuguese (DOS) */
432 {861, "ibm861"}, /* OEM Icelandic; Icelandic (DOS) */
433 {862, "DOS-862"}, /* OEM Hebrew; Hebrew (DOS) */
434 {863, "IBM863"}, /* OEM French Canadian; French Canadian (DOS) */
435 {864, "IBM864"}, /* OEM Arabic; Arabic (864) */
436 {865, "IBM865"}, /* OEM Nordic; Nordic (DOS) */
437 {866, "cp866"}, /* OEM Russian; Cyrillic (DOS) */
438 {869, "ibm869"}, /* OEM Modern Greek; Greek, Modern (DOS) */
439 {870, "IBM870"}, /* IBM EBCDIC Multilingual/ROECE (Latin 2); IBM EBCDIC Multilingual Latin 2 */
440 {874, "windows-874"}, /* ANSI/OEM Thai (same as 28605, ISO 8859-15); Thai (Windows) */
441 {875, "cp875"}, /* IBM EBCDIC Greek Modern */
442 {932, "shift_jis"}, /* ANSI/OEM Japanese; Japanese (Shift-JIS) */
443 {932, "shift-jis"}, /* alternative name for it */
444 {936, "gb2312"}, /* ANSI/OEM Simplified Chinese (PRC, Singapore); Chinese Simplified (GB2312) */
445 {949, "ks_c_5601-1987"}, /* ANSI/OEM Korean (Unified Hangul Code) */
446 {950, "big5"}, /* ANSI/OEM Traditional Chinese (Taiwan; Hong Kong SAR, PRC); Chinese Traditional (Big5) */
447 {950, "big5hkscs"}, /* ANSI/OEM Traditional Chinese (Hong Kong SAR); Chinese Traditional (Big5-HKSCS) */
448 {950, "big5-hkscs"}, /* alternative name for it */
449 {1026, "IBM1026"}, /* IBM EBCDIC Turkish (Latin 5) */
450 {1047, "IBM01047"}, /* IBM EBCDIC Latin 1/Open System */
451 {1140, "IBM01140"}, /* IBM EBCDIC US-Canada (037 + Euro symbol); IBM EBCDIC (US-Canada-Euro) */
452 {1141, "IBM01141"}, /* IBM EBCDIC Germany (20273 + Euro symbol); IBM EBCDIC (Germany-Euro) */
453 {1142, "IBM01142"}, /* IBM EBCDIC Denmark-Norway (20277 + Euro symbol); IBM EBCDIC (Denmark-Norway-Euro) */
454 {1143, "IBM01143"}, /* IBM EBCDIC Finland-Sweden (20278 + Euro symbol); IBM EBCDIC (Finland-Sweden-Euro) */
455 {1144, "IBM01144"}, /* IBM EBCDIC Italy (20280 + Euro symbol); IBM EBCDIC (Italy-Euro) */
456 {1145, "IBM01145"}, /* IBM EBCDIC Latin America-Spain (20284 + Euro symbol); IBM EBCDIC (Spain-Euro) */
457 {1146, "IBM01146"}, /* IBM EBCDIC United Kingdom (20285 + Euro symbol); IBM EBCDIC (UK-Euro) */
458 {1147, "IBM01147"}, /* IBM EBCDIC France (20297 + Euro symbol); IBM EBCDIC (France-Euro) */
459 {1148, "IBM01148"}, /* IBM EBCDIC International (500 + Euro symbol); IBM EBCDIC (International-Euro) */
460 {1149, "IBM01149"}, /* IBM EBCDIC Icelandic (20871 + Euro symbol); IBM EBCDIC (Icelandic-Euro) */
461 {1250, "windows-1250"}, /* ANSI Central European; Central European (Windows) */
462 {1251, "windows-1251"}, /* ANSI Cyrillic; Cyrillic (Windows) */
463 {1252, "windows-1252"}, /* ANSI Latin 1; Western European (Windows) */
464 {1253, "windows-1253"}, /* ANSI Greek; Greek (Windows) */
465 {1254, "windows-1254"}, /* ANSI Turkish; Turkish (Windows) */
466 {1255, "windows-1255"}, /* ANSI Hebrew; Hebrew (Windows) */
467 {1256, "windows-1256"}, /* ANSI Arabic; Arabic (Windows) */
468 {1257, "windows-1257"}, /* ANSI Baltic; Baltic (Windows) */
469 {1258, "windows-1258"}, /* ANSI/OEM Vietnamese; Vietnamese (Windows) */
470 {1361, "Johab"}, /* Korean (Johab) */
471 {10000, "macintosh"}, /* MAC Roman; Western European (Mac) */
472 {10001, "x-mac-japanese"}, /* Japanese (Mac) */
473 {10002, "x-mac-chinesetrad"}, /* MAC Traditional Chinese (Big5); Chinese Traditional (Mac) */
474 {10003, "x-mac-korean"}, /* Korean (Mac) */
475 {10004, "x-mac-arabic"}, /* Arabic (Mac) */
476 {10005, "x-mac-hebrew"}, /* Hebrew (Mac) */
477 {10006, "x-mac-greek"}, /* Greek (Mac) */
478 {10007, "x-mac-cyrillic"}, /* Cyrillic (Mac) */
479 {10008, "x-mac-chinesesimp"}, /* MAC Simplified Chinese (GB 2312); Chinese Simplified (Mac) */
480 {10010, "x-mac-romanian"}, /* Romanian (Mac) */
481 {10017, "x-mac-ukrainian"}, /* Ukrainian (Mac) */
482 {10021, "x-mac-thai"}, /* Thai (Mac) */
483 {10029, "x-mac-ce"}, /* MAC Latin 2; Central European (Mac) */
484 {10079, "x-mac-icelandic"}, /* Icelandic (Mac) */
485 {10081, "x-mac-turkish"}, /* Turkish (Mac) */
486 {10082, "x-mac-croatian"}, /* Croatian (Mac) */
487 {20000, "x-Chinese_CNS"}, /* CNS Taiwan; Chinese Traditional (CNS) */
488 {20001, "x-cp20001"}, /* TCA Taiwan */
489 {20002, "x_Chinese-Eten"}, /* Eten Taiwan; Chinese Traditional (Eten) */
490 {20003, "x-cp20003"}, /* IBM5550 Taiwan */
491 {20004, "x-cp20004"}, /* TeleText Taiwan */
492 {20005, "x-cp20005"}, /* Wang Taiwan */
493 {20105, "x-IA5"}, /* IA5 (IRV International Alphabet No. 5, 7-bit); Western European (IA5) */
494 {20106, "x-IA5-German"}, /* IA5 German (7-bit) */
495 {20107, "x-IA5-Swedish"}, /* IA5 Swedish (7-bit) */
496 {20108, "x-IA5-Norwegian"}, /* IA5 Norwegian (7-bit) */
497 {20127, "us-ascii"}, /* US-ASCII (7-bit) */
498 {20261, "x-cp20261"}, /* T.61 */
499 {20269, "x-cp20269"}, /* ISO 6937 Non-Spacing Accent */
500 {20273, "IBM273"}, /* IBM EBCDIC Germany */
501 {20277, "IBM277"}, /* IBM EBCDIC Denmark-Norway */
502 {20278, "IBM278"}, /* IBM EBCDIC Finland-Sweden */
503 {20280, "IBM280"}, /* IBM EBCDIC Italy */
504 {20284, "IBM284"}, /* IBM EBCDIC Latin America-Spain */
505 {20285, "IBM285"}, /* IBM EBCDIC United Kingdom */
506 {20290, "IBM290"}, /* IBM EBCDIC Japanese Katakana Extended */
507 {20297, "IBM297"}, /* IBM EBCDIC France */
508 {20420, "IBM420"}, /* IBM EBCDIC Arabic */
509 {20423, "IBM423"}, /* IBM EBCDIC Greek */
510 {20424, "IBM424"}, /* IBM EBCDIC Hebrew */
511 {20833, "x-EBCDIC-KoreanExtended"}, /* IBM EBCDIC Korean Extended */
512 {20838, "IBM-Thai"}, /* IBM EBCDIC Thai */
513 {20866, "koi8-r"}, /* Russian (KOI8-R); Cyrillic (KOI8-R) */
514 {20871, "IBM871"}, /* IBM EBCDIC Icelandic */
515 {20880, "IBM880"}, /* IBM EBCDIC Cyrillic Russian */
516 {20905, "IBM905"}, /* IBM EBCDIC Turkish */
517 {20924, "IBM00924"}, /* IBM EBCDIC Latin 1/Open System (1047 + Euro symbol) */
518 {20932, "EUC-JP"}, /* Japanese (JIS 0208-1990 and 0121-1990) */
519 {20936, "x-cp20936"}, /* Simplified Chinese (GB2312); Chinese Simplified (GB2312-80) */
520 {20949, "x-cp20949"}, /* Korean Wansung */
521 {21025, "cp1025"}, /* IBM EBCDIC Cyrillic Serbian-Bulgarian */
522 /* 21027 (deprecated) */
523 {21866, "koi8-u"}, /* Ukrainian (KOI8-U); Cyrillic (KOI8-U) */
524 {28591, "iso-8859-1"}, /* ISO 8859-1 Latin 1; Western European (ISO) */
525 {28591, "iso8859-1"}, /* ISO 8859-1 Latin 1; Western European (ISO) */
526 {28591, "iso_8859-1"},
527 {28591, "iso_8859_1"},
528 {28592, "iso-8859-2"}, /* ISO 8859-2 Central European; Central European (ISO) */
529 {28592, "iso8859-2"}, /* ISO 8859-2 Central European; Central European (ISO) */
530 {28592, "iso_8859-2"},
531 {28592, "iso_8859_2"},
532 {28593, "iso-8859-3"}, /* ISO 8859-3 Latin 3 */
533 {28593, "iso8859-3"}, /* ISO 8859-3 Latin 3 */
534 {28593, "iso_8859-3"},
535 {28593, "iso_8859_3"},
536 {28594, "iso-8859-4"}, /* ISO 8859-4 Baltic */
537 {28594, "iso8859-4"}, /* ISO 8859-4 Baltic */
538 {28594, "iso_8859-4"},
539 {28594, "iso_8859_4"},
540 {28595, "iso-8859-5"}, /* ISO 8859-5 Cyrillic */
541 {28595, "iso8859-5"}, /* ISO 8859-5 Cyrillic */
542 {28595, "iso_8859-5"},
543 {28595, "iso_8859_5"},
544 {28596, "iso-8859-6"}, /* ISO 8859-6 Arabic */
545 {28596, "iso8859-6"}, /* ISO 8859-6 Arabic */
546 {28596, "iso_8859-6"},
547 {28596, "iso_8859_6"},
548 {28597, "iso-8859-7"}, /* ISO 8859-7 Greek */
549 {28597, "iso8859-7"}, /* ISO 8859-7 Greek */
550 {28597, "iso_8859-7"},
551 {28597, "iso_8859_7"},
552 {28598, "iso-8859-8"}, /* ISO 8859-8 Hebrew; Hebrew (ISO-Visual) */
553 {28598, "iso8859-8"}, /* ISO 8859-8 Hebrew; Hebrew (ISO-Visual) */
554 {28598, "iso_8859-8"},
555 {28598, "iso_8859_8"},
556 {28599, "iso-8859-9"}, /* ISO 8859-9 Turkish */
557 {28599, "iso8859-9"}, /* ISO 8859-9 Turkish */
558 {28599, "iso_8859-9"},
559 {28599, "iso_8859_9"},
560 {28603, "iso-8859-13"}, /* ISO 8859-13 Estonian */
561 {28603, "iso8859-13"}, /* ISO 8859-13 Estonian */
562 {28603, "iso_8859-13"},
563 {28603, "iso_8859_13"},
564 {28605, "iso-8859-15"}, /* ISO 8859-15 Latin 9 */
565 {28605, "iso8859-15"}, /* ISO 8859-15 Latin 9 */
566 {28605, "iso_8859-15"},
567 {28605, "iso_8859_15"},
568 {29001, "x-Europa"}, /* Europa 3 */
569 {38598, "iso-8859-8-i"}, /* ISO 8859-8 Hebrew; Hebrew (ISO-Logical) */
570 {38598, "iso8859-8-i"}, /* ISO 8859-8 Hebrew; Hebrew (ISO-Logical) */
571 {38598, "iso_8859-8-i"},
572 {38598, "iso_8859_8-i"},
573 {50220, "iso-2022-jp"}, /* ISO 2022 Japanese with no halfwidth Katakana; Japanese (JIS) */
574 {50221, "csISO2022JP"}, /* ISO 2022 Japanese with halfwidth Katakana; Japanese (JIS-Allow 1 byte Kana) */
575 {50222, "iso-2022-jp"}, /* ISO 2022 Japanese JIS X 0201-1989; Japanese (JIS-Allow 1 byte Kana - SO/SI) */
576 {50225, "iso-2022-kr"}, /* ISO 2022 Korean */
577 {50225, "iso2022-kr"}, /* ISO 2022 Korean */
578 {50227, "x-cp50227"}, /* ISO 2022 Simplified Chinese; Chinese Simplified (ISO 2022) */
579 /* 50229 ISO 2022 Traditional Chinese */
580 /* 50930 EBCDIC Japanese (Katakana) Extended */
581 /* 50931 EBCDIC US-Canada and Japanese */
582 /* 50933 EBCDIC Korean Extended and Korean */
583 /* 50935 EBCDIC Simplified Chinese Extended and Simplified Chinese */
584 /* 50936 EBCDIC Simplified Chinese */
585 /* 50937 EBCDIC US-Canada and Traditional Chinese */
586 /* 50939 EBCDIC Japanese (Latin) Extended and Japanese */
587 {51932, "euc-jp"}, /* EUC Japanese */
588 {51936, "EUC-CN"}, /* EUC Simplified Chinese; Chinese Simplified (EUC) */
589 {51949, "euc-kr"}, /* EUC Korean */
590 /* 51950 EUC Traditional Chinese */
591 {52936, "hz-gb-2312"}, /* HZ-GB2312 Simplified Chinese; Chinese Simplified (HZ) */
592 {54936, "GB18030"}, /* Windows XP and later: GB18030 Simplified Chinese (4 byte); Chinese Simplified (GB18030) */
593 {57002, "x-iscii-de"}, /* ISCII Devanagari */
594 {57003, "x-iscii-be"}, /* ISCII Bengali */
595 {57004, "x-iscii-ta"}, /* ISCII Tamil */
596 {57005, "x-iscii-te"}, /* ISCII Telugu */
597 {57006, "x-iscii-as"}, /* ISCII Assamese */
598 {57007, "x-iscii-or"}, /* ISCII Oriya */
599 {57008, "x-iscii-ka"}, /* ISCII Kannada */
600 {57009, "x-iscii-ma"}, /* ISCII Malayalam */
601 {57010, "x-iscii-gu"}, /* ISCII Gujarati */
602 {57011, "x-iscii-pa"}, /* ISCII Punjabi */
608 * SJIS SHIFTJIS table CP932 table
609 * ---- --------------------------- --------------------------------
610 * 5C U+00A5 YEN SIGN U+005C REVERSE SOLIDUS
611 * 7E U+203E OVERLINE U+007E TILDE
612 * 815C U+2014 EM DASH U+2015 HORIZONTAL BAR
613 * 815F U+005C REVERSE SOLIDUS U+FF3C FULLWIDTH REVERSE SOLIDUS
614 * 8160 U+301C WAVE DASH U+FF5E FULLWIDTH TILDE
615 * 8161 U+2016 DOUBLE VERTICAL LINE U+2225 PARALLEL TO
616 * 817C U+2212 MINUS SIGN U+FF0D FULLWIDTH HYPHEN-MINUS
617 * 8191 U+00A2 CENT SIGN U+FFE0 FULLWIDTH CENT SIGN
618 * 8192 U+00A3 POUND SIGN U+FFE1 FULLWIDTH POUND SIGN
619 * 81CA U+00AC NOT SIGN U+FFE2 FULLWIDTH NOT SIGN
621 * EUC-JP and ISO-2022-JP should be compatible with CP932.
623 * Kernel and MLang have different Unicode mapping table. Make sure
626 static compat_t cp932_compat
[] = {
627 {0x00A5, 0x005C, COMPAT_OUT
},
628 {0x203E, 0x007E, COMPAT_OUT
},
629 {0x2014, 0x2015, COMPAT_OUT
},
630 {0x301C, 0xFF5E, COMPAT_OUT
},
631 {0x2016, 0x2225, COMPAT_OUT
},
632 {0x2212, 0xFF0D, COMPAT_OUT
},
633 {0x00A2, 0xFFE0, COMPAT_OUT
},
634 {0x00A3, 0xFFE1, COMPAT_OUT
},
635 {0x00AC, 0xFFE2, COMPAT_OUT
},
639 static compat_t cp20932_compat
[] = {
640 {0x00A5, 0x005C, COMPAT_OUT
},
641 {0x203E, 0x007E, COMPAT_OUT
},
642 {0x2014, 0x2015, COMPAT_OUT
},
643 {0xFF5E, 0x301C, COMPAT_OUT
|COMPAT_IN
},
644 {0x2225, 0x2016, COMPAT_OUT
|COMPAT_IN
},
645 {0xFF0D, 0x2212, COMPAT_OUT
|COMPAT_IN
},
646 {0xFFE0, 0x00A2, COMPAT_OUT
|COMPAT_IN
},
647 {0xFFE1, 0x00A3, COMPAT_OUT
|COMPAT_IN
},
648 {0xFFE2, 0x00AC, COMPAT_OUT
|COMPAT_IN
},
652 static compat_t
*cp51932_compat
= cp932_compat
;
654 /* cp20932_compat for kernel. cp932_compat for mlang. */
655 static compat_t
*cp5022x_compat
= cp932_compat
;
657 typedef HRESULT (WINAPI
*CONVERTINETSTRING
)(
666 typedef HRESULT (WINAPI
*CONVERTINETMULTIBYTETOUNICODE
)(
670 LPINT lpnMultiCharCount
,
672 LPINT lpnWideCharCount
674 typedef HRESULT (WINAPI
*CONVERTINETUNICODETOMULTIBYTE
)(
678 LPINT lpnWideCharCount
,
680 LPINT lpnMultiCharCount
682 typedef HRESULT (WINAPI
*ISCONVERTINETSTRINGAVAILABLE
)(
686 typedef HRESULT (WINAPI
*LCIDTORFC1766A
)(
691 typedef HRESULT (WINAPI
*LCIDTORFC1766W
)(
696 typedef HRESULT (WINAPI
*RFC1766TOLCIDA
)(
700 typedef HRESULT (WINAPI
*RFC1766TOLCIDW
)(
704 static CONVERTINETSTRING ConvertINetString
;
705 static CONVERTINETMULTIBYTETOUNICODE ConvertINetMultiByteToUnicode
;
706 static CONVERTINETUNICODETOMULTIBYTE ConvertINetUnicodeToMultiByte
;
707 static ISCONVERTINETSTRINGAVAILABLE IsConvertINetStringAvailable
;
708 static LCIDTORFC1766A LcidToRfc1766A
;
709 static RFC1766TOLCIDA Rfc1766ToLcidA
;
715 if (ConvertINetString
!= NULL
)
717 h
= LoadLibrary(TEXT("mlang.dll"));
720 ConvertINetString
= (CONVERTINETSTRING
)GetProcAddressA(h
, "ConvertINetString");
721 ConvertINetMultiByteToUnicode
= (CONVERTINETMULTIBYTETOUNICODE
)GetProcAddressA(h
, "ConvertINetMultiByteToUnicode");
722 ConvertINetUnicodeToMultiByte
= (CONVERTINETUNICODETOMULTIBYTE
)GetProcAddressA(h
, "ConvertINetUnicodeToMultiByte");
723 IsConvertINetStringAvailable
= (ISCONVERTINETSTRINGAVAILABLE
)GetProcAddressA(h
, "IsConvertINetStringAvailable");
724 LcidToRfc1766A
= (LCIDTORFC1766A
)GetProcAddressA(h
, "LcidToRfc1766A");
725 Rfc1766ToLcidA
= (RFC1766TOLCIDA
)GetProcAddressA(h
, "Rfc1766ToLcidA");
730 iconv_open(const char *tocode
, const char *fromcode
)
734 cd
= (rec_iconv_t
*)calloc(1, sizeof(rec_iconv_t
));
736 return (iconv_t
)(-1);
738 #if defined(USE_LIBICONV_DLL)
740 if (libiconv_iconv_open(cd
, tocode
, fromcode
))
744 /* reset the errno to prevent reporting wrong error code.
745 * 0 for unsorted error. */
747 if (win_iconv_open(cd
, tocode
, fromcode
))
752 return (iconv_t
)(-1);
756 iconv_close(iconv_t _cd
)
758 rec_iconv_t
*cd
= (rec_iconv_t
*)_cd
;
759 int r
= cd
->iconv_close(cd
->cd
);
760 int e
= *(cd
->_errno());
761 #if defined(USE_LIBICONV_DLL)
762 if (cd
->hlibiconv
!= NULL
)
763 FreeLibrary(cd
->hlibiconv
);
771 iconv(iconv_t _cd
, const char **inbuf
, size_t *inbytesleft
, char **outbuf
, size_t *outbytesleft
)
773 rec_iconv_t
*cd
= (rec_iconv_t
*)_cd
;
774 size_t r
= cd
->iconv(cd
->cd
, inbuf
, inbytesleft
, outbuf
, outbytesleft
);
775 errno
= *(cd
->_errno());
780 win_iconv_open(rec_iconv_t
*cd
, const char *tocode
, const char *fromcode
)
782 if (!make_csconv(fromcode
, &cd
->from
) || !make_csconv(tocode
, &cd
->to
))
784 cd
->iconv_close
= win_iconv_close
;
785 cd
->iconv
= win_iconv
;
787 cd
->cd
= (iconv_t
)cd
;
792 win_iconv_close(iconv_t cd UNUSED
)
798 win_iconv(iconv_t _cd
, const char **inbuf
, size_t *inbytesleft
, char **outbuf
, size_t *outbytesleft
)
800 rec_iconv_t
*cd
= (rec_iconv_t
*)_cd
;
801 ushort wbuf
[MB_CHAR_MAX
]; /* enough room for one character */
811 if (inbuf
== NULL
|| *inbuf
== NULL
)
813 if (outbuf
!= NULL
&& *outbuf
!= NULL
&& cd
->to
.flush
!= NULL
)
815 tomode
= cd
->to
.mode
;
816 outsize
= cd
->to
.flush(&cd
->to
, (uchar
*)*outbuf
, *outbytesleft
);
819 if ((cd
->to
.flags
& FLAG_IGNORE
) && errno
!= E2BIG
)
825 cd
->to
.mode
= tomode
;
830 *outbytesleft
-= outsize
;
837 while (*inbytesleft
!= 0)
839 frommode
= cd
->from
.mode
;
840 tomode
= cd
->to
.mode
;
843 insize
= cd
->from
.mbtowc(&cd
->from
, (const uchar
*)*inbuf
, *inbytesleft
, wbuf
, &wsize
);
846 if (cd
->to
.flags
& FLAG_IGNORE
)
848 cd
->from
.mode
= frommode
;
854 cd
->from
.mode
= frommode
;
862 *inbytesleft
-= insize
;
866 if (cd
->from
.compat
!= NULL
)
868 wc
= utf16_to_ucs4(wbuf
);
869 cp
= cd
->from
.compat
;
870 for (i
= 0; cp
[i
].in
!= 0; ++i
)
872 if ((cp
[i
].flag
& COMPAT_IN
) && cp
[i
].out
== wc
)
874 ucs4_to_utf16(cp
[i
].in
, wbuf
, &wsize
);
880 if (cd
->to
.compat
!= NULL
)
882 wc
= utf16_to_ucs4(wbuf
);
884 for (i
= 0; cp
[i
].in
!= 0; ++i
)
886 if ((cp
[i
].flag
& COMPAT_OUT
) && cp
[i
].in
== wc
)
888 ucs4_to_utf16(cp
[i
].out
, wbuf
, &wsize
);
894 outsize
= cd
->to
.wctomb(&cd
->to
, wbuf
, wsize
, (uchar
*)*outbuf
, *outbytesleft
);
897 if ((cd
->to
.flags
& FLAG_IGNORE
) && errno
!= E2BIG
)
899 cd
->to
.mode
= tomode
;
904 cd
->from
.mode
= frommode
;
905 cd
->to
.mode
= tomode
;
912 *inbytesleft
-= insize
;
913 *outbytesleft
-= outsize
;
920 make_csconv(const char *_name
, csconv_t
*cv
)
923 int use_compat
= TRUE
;
928 name
= xstrndup(_name
, strlen(_name
));
932 /* check for option "enc_name//opt1//opt2" */
933 while ((p
= strrstr(name
, "//")) != NULL
)
935 if (_stricmp(p
+ 2, "nocompat") == 0)
937 else if (_stricmp(p
+ 2, "translit") == 0)
938 flag
|= FLAG_TRANSLIT
;
939 else if (_stricmp(p
+ 2, "ignore") == 0)
949 cv
->codepage
= name_to_codepage(name
);
950 if (cv
->codepage
== 1200 || cv
->codepage
== 1201)
952 cv
->mbtowc
= utf16_mbtowc
;
953 cv
->wctomb
= utf16_wctomb
;
954 if (_stricmp(name
, "UTF-16") == 0 || _stricmp(name
, "UTF16") == 0 ||
955 _stricmp(name
, "UCS-2") == 0 || _stricmp(name
, "UCS2") == 0)
956 cv
->flags
|= FLAG_USE_BOM
;
958 else if (cv
->codepage
== 12000 || cv
->codepage
== 12001)
960 cv
->mbtowc
= utf32_mbtowc
;
961 cv
->wctomb
= utf32_wctomb
;
962 if (_stricmp(name
, "UTF-32") == 0 || _stricmp(name
, "UTF32") == 0 ||
963 _stricmp(name
, "UCS-4") == 0 || _stricmp(name
, "UCS4") == 0)
964 cv
->flags
|= FLAG_USE_BOM
;
966 else if (cv
->codepage
== 65001)
968 cv
->mbtowc
= kernel_mbtowc
;
969 cv
->wctomb
= kernel_wctomb
;
970 cv
->mblen
= utf8_mblen
;
972 else if ((cv
->codepage
== 50220 || cv
->codepage
== 50221 || cv
->codepage
== 50222) && load_mlang())
974 cv
->mbtowc
= iso2022jp_mbtowc
;
975 cv
->wctomb
= iso2022jp_wctomb
;
976 cv
->flush
= iso2022jp_flush
;
978 else if (cv
->codepage
== 51932 && load_mlang())
980 cv
->mbtowc
= mlang_mbtowc
;
981 cv
->wctomb
= mlang_wctomb
;
982 cv
->mblen
= eucjp_mblen
;
984 else if (IsValidCodePage(cv
->codepage
)
985 && GetCPInfo(cv
->codepage
, &cpinfo
) != 0)
987 cv
->mbtowc
= kernel_mbtowc
;
988 cv
->wctomb
= kernel_wctomb
;
989 if (cpinfo
.MaxCharSize
== 1)
990 cv
->mblen
= sbcs_mblen
;
991 else if (cpinfo
.MaxCharSize
== 2)
992 cv
->mblen
= dbcs_mblen
;
994 cv
->mblen
= mbcs_mblen
;
1006 switch (cv
->codepage
)
1008 case 932: cv
->compat
= cp932_compat
; break;
1009 case 20932: cv
->compat
= cp20932_compat
; break;
1010 case 51932: cv
->compat
= cp51932_compat
; break;
1011 case 50220: case 50221: case 50222: cv
->compat
= cp5022x_compat
; break;
1021 name_to_codepage(const char *name
)
1025 if (*name
== '\0' ||
1026 strcmp(name
, "char") == 0)
1028 else if (strcmp(name
, "wchar_t") == 0)
1030 else if (_strnicmp(name
, "cp", 2) == 0)
1031 return atoi(name
+ 2); /* CP123 */
1032 else if ('0' <= name
[0] && name
[0] <= '9')
1033 return atoi(name
); /* 123 */
1034 else if (_strnicmp(name
, "xx", 2) == 0)
1035 return atoi(name
+ 2); /* XX123 for debug */
1037 for (i
= 0; codepage_alias
[i
].name
!= NULL
; ++i
)
1038 if (_stricmp(name
, codepage_alias
[i
].name
) == 0)
1039 return codepage_alias
[i
].codepage
;
1044 * http://www.faqs.org/rfcs/rfc2781.html
1047 utf16_to_ucs4(const ushort
*wbuf
)
1050 if (0xD800 <= wbuf
[0] && wbuf
[0] <= 0xDBFF)
1051 wc
= ((wbuf
[0] & 0x3FF) << 10) + (wbuf
[1] & 0x3FF) + 0x10000;
1056 ucs4_to_utf16(uint wc
, ushort
*wbuf
, int *wbufsize
)
1066 wbuf
[0] = 0xD800 | ((wc
>> 10) & 0x3FF);
1067 wbuf
[1] = 0xDC00 | (wc
& 0x3FF);
1073 * Check if codepage is one of those for which the dwFlags parameter
1074 * to MultiByteToWideChar() must be zero. Return zero or
1075 * MB_ERR_INVALID_CHARS. The docs in Platform SDK for for Windows
1076 * Server 2003 R2 claims that also codepage 65001 is one of these, but
1077 * that doesn't seem to be the case. The MSDN docs for MSVS2008 leave
1078 * out 65001 (UTF-8), and that indeed seems to be the case on XP, it
1079 * works fine to pass MB_ERR_INVALID_CHARS in dwFlags when converting
1083 mbtowc_flags(int codepage
)
1085 return (codepage
== 50220 || codepage
== 50221 ||
1086 codepage
== 50222 || codepage
== 50225 ||
1087 codepage
== 50227 || codepage
== 50229 ||
1088 codepage
== 52936 || codepage
== 54936 ||
1089 (codepage
>= 57002 && codepage
<= 57011) ||
1090 codepage
== 65000 || codepage
== 42) ? 0 : MB_ERR_INVALID_CHARS
;
1094 * Check if codepage is one those for which the lpUsedDefaultChar
1095 * parameter to WideCharToMultiByte() must be NULL. The docs in
1096 * Platform SDK for for Windows Server 2003 R2 claims that this is the
1097 * list below, while the MSDN docs for MSVS2008 claim that it is only
1098 * for 65000 (UTF-7) and 65001 (UTF-8). This time the earlier Platform
1099 * SDK seems to be correct, at least for XP.
1102 must_use_null_useddefaultchar(int codepage
)
1104 return (codepage
== 65000 || codepage
== 65001 ||
1105 codepage
== 50220 || codepage
== 50221 ||
1106 codepage
== 50222 || codepage
== 50225 ||
1107 codepage
== 50227 || codepage
== 50229 ||
1108 codepage
== 52936 || codepage
== 54936 ||
1109 (codepage
>= 57002 && codepage
<= 57011) ||
1114 strrstr(const char *str
, const char *token
)
1116 int len
= strlen(token
);
1117 const char *p
= str
+ strlen(str
);
1120 if (p
[0] == token
[0] && strncmp(p
, token
, len
) == 0)
1126 xstrndup(const char *s
, size_t n
)
1130 p
= (char *)malloc(n
+ 1);
1145 #if defined(USE_LIBICONV_DLL)
1147 libiconv_iconv_open(rec_iconv_t
*cd
, const char *tocode
, const char *fromcode
)
1149 HMODULE hlibiconv
= NULL
;
1150 HMODULE hmsvcrt
= NULL
;
1154 f_iconv_open _iconv_open
;
1157 * always try to load dll, so that we can switch dll in runtime.
1160 /* XXX: getenv() can't get variable set by SetEnvironmentVariable() */
1161 p
= getenv("WINICONV_LIBICONV_DLL");
1163 p
= DEFAULT_LIBICONV_DLL
;
1164 /* parse comma separated value */
1165 for ( ; *p
!= 0; p
= (*e
== ',') ? e
+ 1 : e
)
1172 dllname
= xstrndup(p
, e
- p
);
1173 if (dllname
== NULL
)
1175 hlibiconv
= LoadLibraryA(dllname
);
1177 if (hlibiconv
!= NULL
)
1179 if (hlibiconv
== hwiniconv
)
1181 FreeLibrary(hlibiconv
);
1189 if (hlibiconv
== NULL
)
1192 hmsvcrt
= find_imported_module_by_funcname(hlibiconv
, "_errno");
1193 if (hmsvcrt
== NULL
)
1196 _iconv_open
= (f_iconv_open
)GetProcAddressA(hlibiconv
, "libiconv_open");
1197 if (_iconv_open
== NULL
)
1198 _iconv_open
= (f_iconv_open
)GetProcAddressA(hlibiconv
, "iconv_open");
1199 cd
->iconv_close
= (f_iconv_close
)GetProcAddressA(hlibiconv
, "libiconv_close");
1200 if (cd
->iconv_close
== NULL
)
1201 cd
->iconv_close
= (f_iconv_close
)GetProcAddressA(hlibiconv
, "iconv_close");
1202 cd
->iconv
= (f_iconv
)GetProcAddressA(hlibiconv
, "libiconv");
1203 if (cd
->iconv
== NULL
)
1204 cd
->iconv
= (f_iconv
)GetProcAddressA(hlibiconv
, "iconv");
1205 cd
->_errno
= (f_errno
)GetProcAddressA(hmsvcrt
, "_errno");
1206 if (_iconv_open
== NULL
|| cd
->iconv_close
== NULL
1207 || cd
->iconv
== NULL
|| cd
->_errno
== NULL
)
1210 cd
->cd
= _iconv_open(tocode
, fromcode
);
1211 if (cd
->cd
== (iconv_t
)(-1))
1214 cd
->hlibiconv
= hlibiconv
;
1218 if (hlibiconv
!= NULL
)
1219 FreeLibrary(hlibiconv
);
1220 /* do not free hmsvcrt which is obtained by GetModuleHandle() */
1226 * http://forums.belution.com/ja/vc/000/234/78s.shtml
1227 * http://nienie.com/~masapico/api_ImageDirectoryEntryToData.html
1230 * imagehlp.h or dbghelp.h
1231 * imagehlp.lib or dbghelp.lib
1232 * ImageDirectoryEntryToData()
1234 #define TO_DOS_HEADER(base) ((PIMAGE_DOS_HEADER)(base))
1235 #define TO_NT_HEADERS(base) ((PIMAGE_NT_HEADERS)((LPBYTE)(base) + TO_DOS_HEADER(base)->e_lfanew))
1237 MyImageDirectoryEntryToData(LPVOID Base
, BOOLEAN MappedAsImage
, USHORT DirectoryEntry
, PULONG Size
)
1239 /* TODO: MappedAsImage? */
1240 PIMAGE_DATA_DIRECTORY p
;
1241 p
= TO_NT_HEADERS(Base
)->OptionalHeader
.DataDirectory
+ DirectoryEntry
;
1242 if (p
->VirtualAddress
== 0) {
1247 return (PVOID
)((LPBYTE
)Base
+ p
->VirtualAddress
);
1251 find_imported_module_by_funcname(HMODULE hModule
, const char *funcname
)
1255 PIMAGE_IMPORT_DESCRIPTOR Imp
;
1256 PIMAGE_THUNK_DATA Name
; /* Import Name Table */
1257 PIMAGE_IMPORT_BY_NAME ImpName
;
1259 Base
= (DWORD_PTR
)hModule
;
1260 Imp
= (PIMAGE_IMPORT_DESCRIPTOR
)MyImageDirectoryEntryToData(
1263 IMAGE_DIRECTORY_ENTRY_IMPORT
,
1267 for ( ; Imp
->OriginalFirstThunk
!= 0; ++Imp
)
1269 Name
= (PIMAGE_THUNK_DATA
)(Base
+ Imp
->OriginalFirstThunk
);
1270 for ( ; Name
->u1
.Ordinal
!= 0; ++Name
)
1272 if (!IMAGE_SNAP_BY_ORDINAL(Name
->u1
.Ordinal
))
1274 ImpName
= (PIMAGE_IMPORT_BY_NAME
)
1275 (Base
+ (DWORD_PTR
)Name
->u1
.AddressOfData
);
1276 if (strcmp((char *)ImpName
->Name
, funcname
) == 0)
1277 return GetModuleHandleA((char *)(Base
+ Imp
->Name
));
1286 sbcs_mblen(csconv_t
*cv UNUSED
, const uchar
*buf UNUSED
, int bufsize UNUSED
)
1292 dbcs_mblen(csconv_t
*cv
, const uchar
*buf
, int bufsize
)
1294 int len
= IsDBCSLeadByteEx(cv
->codepage
, buf
[0]) ? 2 : 1;
1296 return seterror(EINVAL
);
1301 mbcs_mblen(csconv_t
*cv
, const uchar
*buf
, int bufsize
)
1305 if (cv
->codepage
== 54936) {
1306 if (buf
[0] <= 0x7F) len
= 1;
1307 else if (buf
[0] >= 0x81 && buf
[0] <= 0xFE &&
1309 ((buf
[1] >= 0x40 && buf
[1] <= 0x7E) ||
1310 (buf
[1] >= 0x80 && buf
[1] <= 0xFE))) len
= 2;
1311 else if (buf
[0] >= 0x81 && buf
[0] <= 0xFE &&
1313 buf
[1] >= 0x30 && buf
[1] <= 0x39) len
= 4;
1315 return seterror(EINVAL
);
1319 return seterror(EINVAL
);
1323 utf8_mblen(csconv_t
*cv UNUSED
, const uchar
*buf
, int bufsize
)
1327 if (buf
[0] < 0x80) len
= 1;
1328 else if ((buf
[0] & 0xE0) == 0xC0) len
= 2;
1329 else if ((buf
[0] & 0xF0) == 0xE0) len
= 3;
1330 else if ((buf
[0] & 0xF8) == 0xF0) len
= 4;
1331 else if ((buf
[0] & 0xFC) == 0xF8) len
= 5;
1332 else if ((buf
[0] & 0xFE) == 0xFC) len
= 6;
1335 return seterror(EILSEQ
);
1336 else if (bufsize
< len
)
1337 return seterror(EINVAL
);
1342 eucjp_mblen(csconv_t
*cv UNUSED
, const uchar
*buf
, int bufsize
)
1344 if (buf
[0] < 0x80) /* ASCII */
1346 else if (buf
[0] == 0x8E) /* JIS X 0201 */
1349 return seterror(EINVAL
);
1350 else if (!(0xA1 <= buf
[1] && buf
[1] <= 0xDF))
1351 return seterror(EILSEQ
);
1354 else if (buf
[0] == 0x8F) /* JIS X 0212 */
1357 return seterror(EINVAL
);
1358 else if (!(0xA1 <= buf
[1] && buf
[1] <= 0xFE)
1359 || !(0xA1 <= buf
[2] && buf
[2] <= 0xFE))
1360 return seterror(EILSEQ
);
1363 else /* JIS X 0208 */
1366 return seterror(EINVAL
);
1367 else if (!(0xA1 <= buf
[0] && buf
[0] <= 0xFE)
1368 || !(0xA1 <= buf
[1] && buf
[1] <= 0xFE))
1369 return seterror(EILSEQ
);
1375 kernel_mbtowc(csconv_t
*cv
, const uchar
*buf
, int bufsize
, ushort
*wbuf
, int *wbufsize
)
1379 len
= cv
->mblen(cv
, buf
, bufsize
);
1382 *wbufsize
= MultiByteToWideChar(cv
->codepage
, mbtowc_flags (cv
->codepage
),
1383 (const char *)buf
, len
, (wchar_t *)wbuf
, *wbufsize
);
1385 return seterror(EILSEQ
);
1390 kernel_wctomb(csconv_t
*cv
, ushort
*wbuf
, int wbufsize
, uchar
*buf
, int bufsize
)
1392 BOOL usedDefaultChar
= 0;
1398 return seterror(E2BIG
);
1399 if (!must_use_null_useddefaultchar(cv
->codepage
))
1401 p
= &usedDefaultChar
;
1402 #ifdef WC_NO_BEST_FIT_CHARS
1403 if (!(cv
->flags
& FLAG_TRANSLIT
))
1404 flags
|= WC_NO_BEST_FIT_CHARS
;
1407 len
= WideCharToMultiByte(cv
->codepage
, flags
,
1408 (const wchar_t *)wbuf
, wbufsize
, (char *)buf
, bufsize
, NULL
, p
);
1411 if (GetLastError() == ERROR_INSUFFICIENT_BUFFER
)
1412 return seterror(E2BIG
);
1413 return seterror(EILSEQ
);
1415 else if (usedDefaultChar
&& !(cv
->flags
& FLAG_TRANSLIT
))
1416 return seterror(EILSEQ
);
1417 else if (cv
->mblen(cv
, buf
, len
) != len
) /* validate result */
1418 return seterror(EILSEQ
);
1423 * It seems that the mode (cv->mode) is fixnum.
1424 * For example, when converting iso-2022-jp(cp50221) to unicode:
1425 * in ascii sequence: mode=0xC42C0000
1426 * in jisx0208 sequence: mode=0xC42C0001
1427 * "C42C" is same for each convert session.
1428 * It should be: ((codepage-1)<<16)|state
1431 mlang_mbtowc(csconv_t
*cv
, const uchar
*buf
, int bufsize
, ushort
*wbuf
, int *wbufsize
)
1437 len
= cv
->mblen(cv
, buf
, bufsize
);
1441 hr
= ConvertINetMultiByteToUnicode(&cv
->mode
, cv
->codepage
,
1442 (const char *)buf
, &insize
, (wchar_t *)wbuf
, wbufsize
);
1443 if (hr
!= S_OK
|| insize
!= len
)
1444 return seterror(EILSEQ
);
1449 mlang_wctomb(csconv_t
*cv
, ushort
*wbuf
, int wbufsize
, uchar
*buf
, int bufsize
)
1451 char tmpbuf
[MB_CHAR_MAX
]; /* enough room for one character */
1452 int tmpsize
= MB_CHAR_MAX
;
1453 int insize
= wbufsize
;
1456 hr
= ConvertINetUnicodeToMultiByte(&cv
->mode
, cv
->codepage
,
1457 (const wchar_t *)wbuf
, &wbufsize
, tmpbuf
, &tmpsize
);
1458 if (hr
!= S_OK
|| insize
!= wbufsize
)
1459 return seterror(EILSEQ
);
1460 else if (bufsize
< tmpsize
)
1461 return seterror(E2BIG
);
1462 else if (cv
->mblen(cv
, (uchar
*)tmpbuf
, tmpsize
) != tmpsize
)
1463 return seterror(EILSEQ
);
1464 memcpy(buf
, tmpbuf
, tmpsize
);
1469 utf16_mbtowc(csconv_t
*cv
, const uchar
*buf
, int bufsize
, ushort
*wbuf
, int *wbufsize
)
1471 int codepage
= cv
->codepage
;
1473 /* swap endian: 1200 <-> 1201 */
1474 if (cv
->mode
& UNICODE_MODE_SWAPPED
)
1478 return seterror(EINVAL
);
1479 if (codepage
== 1200) /* little endian */
1480 wbuf
[0] = (buf
[1] << 8) | buf
[0];
1481 else if (codepage
== 1201) /* big endian */
1482 wbuf
[0] = (buf
[0] << 8) | buf
[1];
1484 if ((cv
->flags
& FLAG_USE_BOM
) && !(cv
->mode
& UNICODE_MODE_BOM_DONE
))
1486 cv
->mode
|= UNICODE_MODE_BOM_DONE
;
1487 if (wbuf
[0] == 0xFFFE)
1489 cv
->mode
|= UNICODE_MODE_SWAPPED
;
1493 else if (wbuf
[0] == 0xFEFF)
1500 if (0xDC00 <= wbuf
[0] && wbuf
[0] <= 0xDFFF)
1501 return seterror(EILSEQ
);
1502 if (0xD800 <= wbuf
[0] && wbuf
[0] <= 0xDBFF)
1505 return seterror(EINVAL
);
1506 if (codepage
== 1200) /* little endian */
1507 wbuf
[1] = (buf
[3] << 8) | buf
[2];
1508 else if (codepage
== 1201) /* big endian */
1509 wbuf
[1] = (buf
[2] << 8) | buf
[3];
1510 if (!(0xDC00 <= wbuf
[1] && wbuf
[1] <= 0xDFFF))
1511 return seterror(EILSEQ
);
1520 utf16_wctomb(csconv_t
*cv
, ushort
*wbuf
, int wbufsize
, uchar
*buf
, int bufsize
)
1522 if ((cv
->flags
& FLAG_USE_BOM
) && !(cv
->mode
& UNICODE_MODE_BOM_DONE
))
1526 cv
->mode
|= UNICODE_MODE_BOM_DONE
;
1528 return seterror(E2BIG
);
1529 if (cv
->codepage
== 1200) /* little endian */
1530 memcpy(buf
, "\xFF\xFE", 2);
1531 else if (cv
->codepage
== 1201) /* big endian */
1532 memcpy(buf
, "\xFE\xFF", 2);
1534 r
= utf16_wctomb(cv
, wbuf
, wbufsize
, buf
+ 2, bufsize
- 2);
1541 return seterror(E2BIG
);
1542 if (cv
->codepage
== 1200) /* little endian */
1544 buf
[0] = (wbuf
[0] & 0x00FF);
1545 buf
[1] = (wbuf
[0] & 0xFF00) >> 8;
1547 else if (cv
->codepage
== 1201) /* big endian */
1549 buf
[0] = (wbuf
[0] & 0xFF00) >> 8;
1550 buf
[1] = (wbuf
[0] & 0x00FF);
1552 if (0xD800 <= wbuf
[0] && wbuf
[0] <= 0xDBFF)
1555 return seterror(E2BIG
);
1556 if (cv
->codepage
== 1200) /* little endian */
1558 buf
[2] = (wbuf
[1] & 0x00FF);
1559 buf
[3] = (wbuf
[1] & 0xFF00) >> 8;
1561 else if (cv
->codepage
== 1201) /* big endian */
1563 buf
[2] = (wbuf
[1] & 0xFF00) >> 8;
1564 buf
[3] = (wbuf
[1] & 0x00FF);
1572 utf32_mbtowc(csconv_t
*cv
, const uchar
*buf
, int bufsize
, ushort
*wbuf
, int *wbufsize
)
1574 int codepage
= cv
->codepage
;
1577 /* swap endian: 12000 <-> 12001 */
1578 if (cv
->mode
& UNICODE_MODE_SWAPPED
)
1582 return seterror(EINVAL
);
1583 if (codepage
== 12000) /* little endian */
1584 wc
= (buf
[3] << 24) | (buf
[2] << 16) | (buf
[1] << 8) | buf
[0];
1585 else if (codepage
== 12001) /* big endian */
1586 wc
= (buf
[0] << 24) | (buf
[1] << 16) | (buf
[2] << 8) | buf
[3];
1588 if ((cv
->flags
& FLAG_USE_BOM
) && !(cv
->mode
& UNICODE_MODE_BOM_DONE
))
1590 cv
->mode
|= UNICODE_MODE_BOM_DONE
;
1591 if (wc
== 0xFFFE0000)
1593 cv
->mode
|= UNICODE_MODE_SWAPPED
;
1597 else if (wc
== 0x0000FEFF)
1604 if ((0xD800 <= wc
&& wc
<= 0xDFFF) || 0x10FFFF < wc
)
1605 return seterror(EILSEQ
);
1606 ucs4_to_utf16(wc
, wbuf
, wbufsize
);
1611 utf32_wctomb(csconv_t
*cv
, ushort
*wbuf
, int wbufsize
, uchar
*buf
, int bufsize
)
1615 if ((cv
->flags
& FLAG_USE_BOM
) && !(cv
->mode
& UNICODE_MODE_BOM_DONE
))
1619 cv
->mode
|= UNICODE_MODE_BOM_DONE
;
1621 return seterror(E2BIG
);
1622 if (cv
->codepage
== 12000) /* little endian */
1623 memcpy(buf
, "\xFF\xFE\x00\x00", 4);
1624 else if (cv
->codepage
== 12001) /* big endian */
1625 memcpy(buf
, "\x00\x00\xFE\xFF", 4);
1627 r
= utf32_wctomb(cv
, wbuf
, wbufsize
, buf
+ 4, bufsize
- 4);
1634 return seterror(E2BIG
);
1635 wc
= utf16_to_ucs4(wbuf
);
1636 if (cv
->codepage
== 12000) /* little endian */
1638 buf
[0] = wc
& 0x000000FF;
1639 buf
[1] = (wc
& 0x0000FF00) >> 8;
1640 buf
[2] = (wc
& 0x00FF0000) >> 16;
1641 buf
[3] = (wc
& 0xFF000000) >> 24;
1643 else if (cv
->codepage
== 12001) /* big endian */
1645 buf
[0] = (wc
& 0xFF000000) >> 24;
1646 buf
[1] = (wc
& 0x00FF0000) >> 16;
1647 buf
[2] = (wc
& 0x0000FF00) >> 8;
1648 buf
[3] = wc
& 0x000000FF;
1654 * 50220: ISO 2022 Japanese with no halfwidth Katakana; Japanese (JIS)
1655 * 50221: ISO 2022 Japanese with halfwidth Katakana; Japanese (JIS-Allow
1657 * 50222: ISO 2022 Japanese JIS X 0201-1989; Japanese (JIS-Allow 1 byte
1660 * MultiByteToWideChar() and WideCharToMultiByte() behave differently
1661 * depending on Windows version. On XP, WideCharToMultiByte() doesn't
1662 * terminate result sequence with ascii escape. But Vista does.
1663 * Use MLang instead.
1666 #define ISO2022_MODE(cs, shift) (((cs) << 8) | (shift))
1667 #define ISO2022_MODE_CS(mode) (((mode) >> 8) & 0xFF)
1668 #define ISO2022_MODE_SHIFT(mode) ((mode) & 0xFF)
1670 #define ISO2022_SI 0
1671 #define ISO2022_SO 1
1674 static const char iso2022_SI_seq
[] = "\x0F";
1676 static const char iso2022_SO_seq
[] = "\x0E";
1678 typedef struct iso2022_esc_t iso2022_esc_t
;
1679 struct iso2022_esc_t
{
1686 #define ISO2022JP_CS_ASCII 0
1687 #define ISO2022JP_CS_JISX0201_ROMAN 1
1688 #define ISO2022JP_CS_JISX0201_KANA 2
1689 #define ISO2022JP_CS_JISX0208_1978 3
1690 #define ISO2022JP_CS_JISX0208_1983 4
1691 #define ISO2022JP_CS_JISX0212 5
1693 static iso2022_esc_t iso2022jp_esc
[] = {
1694 {"\x1B\x28\x42", 3, 1, ISO2022JP_CS_ASCII
},
1695 {"\x1B\x28\x4A", 3, 1, ISO2022JP_CS_JISX0201_ROMAN
},
1696 {"\x1B\x28\x49", 3, 1, ISO2022JP_CS_JISX0201_KANA
},
1697 {"\x1B\x24\x40", 3, 2, ISO2022JP_CS_JISX0208_1983
}, /* unify 1978 with 1983 */
1698 {"\x1B\x24\x42", 3, 2, ISO2022JP_CS_JISX0208_1983
},
1699 {"\x1B\x24\x28\x44", 4, 2, ISO2022JP_CS_JISX0212
},
1704 iso2022jp_mbtowc(csconv_t
*cv
, const uchar
*buf
, int bufsize
, ushort
*wbuf
, int *wbufsize
)
1706 iso2022_esc_t
*iesc
= iso2022jp_esc
;
1707 char tmp
[MB_CHAR_MAX
];
1719 for (i
= 0; iesc
[i
].esc
!= NULL
; ++i
)
1721 esc_len
= iesc
[i
].esc_len
;
1722 if (bufsize
< esc_len
)
1724 if (strncmp((char *)buf
, iesc
[i
].esc
, bufsize
) == 0)
1725 return seterror(EINVAL
);
1729 if (strncmp((char *)buf
, iesc
[i
].esc
, esc_len
) == 0)
1731 cv
->mode
= ISO2022_MODE(iesc
[i
].cs
, ISO2022_SI
);
1737 /* not supported escape sequence */
1738 return seterror(EILSEQ
);
1740 else if (buf
[0] == iso2022_SO_seq
[0])
1742 cv
->mode
= ISO2022_MODE(ISO2022_MODE_CS(cv
->mode
), ISO2022_SO
);
1746 else if (buf
[0] == iso2022_SI_seq
[0])
1748 cv
->mode
= ISO2022_MODE(ISO2022_MODE_CS(cv
->mode
), ISO2022_SI
);
1753 cs
= ISO2022_MODE_CS(cv
->mode
);
1754 shift
= ISO2022_MODE_SHIFT(cv
->mode
);
1756 /* reset the mode for informal sequence */
1759 cs
= ISO2022JP_CS_ASCII
;
1765 return seterror(EINVAL
);
1766 for (i
= 0; i
< len
; ++i
)
1767 if (!(buf
[i
] < 0x80))
1768 return seterror(EILSEQ
);
1769 esc_len
= iesc
[cs
].esc_len
;
1770 memcpy(tmp
, iesc
[cs
].esc
, esc_len
);
1771 if (shift
== ISO2022_SO
)
1773 memcpy(tmp
+ esc_len
, iso2022_SO_seq
, 1);
1776 memcpy(tmp
+ esc_len
, buf
, len
);
1778 if ((cv
->codepage
== 50220 || cv
->codepage
== 50221
1779 || cv
->codepage
== 50222) && shift
== ISO2022_SO
)
1781 /* XXX: shift-out cannot be used for mbtowc (both kernel and
1783 esc_len
= iesc
[ISO2022JP_CS_JISX0201_KANA
].esc_len
;
1784 memcpy(tmp
, iesc
[ISO2022JP_CS_JISX0201_KANA
].esc
, esc_len
);
1785 memcpy(tmp
+ esc_len
, buf
, len
);
1788 insize
= len
+ esc_len
;
1789 hr
= ConvertINetMultiByteToUnicode(&dummy
, cv
->codepage
,
1790 (const char *)tmp
, &insize
, (wchar_t *)wbuf
, wbufsize
);
1791 if (hr
!= S_OK
|| insize
!= len
+ esc_len
)
1792 return seterror(EILSEQ
);
1794 /* Check for conversion error. Assuming defaultChar is 0x3F. */
1795 /* ascii should be converted from ascii */
1796 if (wbuf
[0] == buf
[0]
1797 && cv
->mode
!= ISO2022_MODE(ISO2022JP_CS_ASCII
, ISO2022_SI
))
1798 return seterror(EILSEQ
);
1800 /* reset the mode for informal sequence */
1801 if (cv
->mode
!= ISO2022_MODE(cs
, shift
))
1802 cv
->mode
= ISO2022_MODE(cs
, shift
);
1808 iso2022jp_wctomb(csconv_t
*cv
, ushort
*wbuf
, int wbufsize
, uchar
*buf
, int bufsize
)
1810 iso2022_esc_t
*iesc
= iso2022jp_esc
;
1811 char tmp
[MB_CHAR_MAX
];
1812 int tmpsize
= MB_CHAR_MAX
;
1813 int insize
= wbufsize
;
1823 * MultiByte = [escape sequence] + character + [escape sequence]
1825 * Whether trailing escape sequence is added depends on which API is
1826 * used (kernel or MLang, and its version).
1828 hr
= ConvertINetUnicodeToMultiByte(&dummy
, cv
->codepage
,
1829 (const wchar_t *)wbuf
, &wbufsize
, tmp
, &tmpsize
);
1830 if (hr
!= S_OK
|| insize
!= wbufsize
)
1831 return seterror(EILSEQ
);
1832 else if (bufsize
< tmpsize
)
1833 return seterror(E2BIG
);
1837 cs
= ISO2022JP_CS_ASCII
;
1842 for (i
= 1; iesc
[i
].esc
!= NULL
; ++i
)
1844 esc_len
= iesc
[i
].esc_len
;
1845 if (strncmp(tmp
, iesc
[i
].esc
, esc_len
) == 0)
1851 if (iesc
[i
].esc
== NULL
)
1852 /* not supported escape sequence */
1853 return seterror(EILSEQ
);
1857 if (tmp
[esc_len
] == iso2022_SO_seq
[0])
1865 /* Check for converting error. Assuming defaultChar is 0x3F. */
1866 /* ascii should be converted from ascii */
1867 if (cs
== ISO2022JP_CS_ASCII
&& !(wbuf
[0] < 0x80))
1868 return seterror(EILSEQ
);
1869 else if (tmpsize
< esc_len
+ len
)
1870 return seterror(EILSEQ
);
1872 if (cv
->mode
== ISO2022_MODE(cs
, shift
))
1874 /* remove escape sequence */
1876 memmove(tmp
, tmp
+ esc_len
, len
);
1881 if (cs
== ISO2022JP_CS_ASCII
)
1883 esc_len
= iesc
[ISO2022JP_CS_ASCII
].esc_len
;
1884 memmove(tmp
+ esc_len
, tmp
, len
);
1885 memcpy(tmp
, iesc
[ISO2022JP_CS_ASCII
].esc
, esc_len
);
1887 if (ISO2022_MODE_SHIFT(cv
->mode
) == ISO2022_SO
)
1889 /* shift-in before changing to other mode */
1890 memmove(tmp
+ 1, tmp
, len
+ esc_len
);
1891 memcpy(tmp
, iso2022_SI_seq
, 1);
1896 if (bufsize
< len
+ esc_len
)
1897 return seterror(E2BIG
);
1898 memcpy(buf
, tmp
, len
+ esc_len
);
1899 cv
->mode
= ISO2022_MODE(cs
, shift
);
1900 return len
+ esc_len
;
1904 iso2022jp_flush(csconv_t
*cv
, uchar
*buf
, int bufsize
)
1906 iso2022_esc_t
*iesc
= iso2022jp_esc
;
1909 if (cv
->mode
!= ISO2022_MODE(ISO2022JP_CS_ASCII
, ISO2022_SI
))
1912 if (ISO2022_MODE_SHIFT(cv
->mode
) != ISO2022_SI
)
1914 if (ISO2022_MODE_CS(cv
->mode
) != ISO2022JP_CS_ASCII
)
1915 esc_len
+= iesc
[ISO2022JP_CS_ASCII
].esc_len
;
1916 if (bufsize
< esc_len
)
1917 return seterror(E2BIG
);
1920 if (ISO2022_MODE_SHIFT(cv
->mode
) != ISO2022_SI
)
1922 memcpy(buf
, iso2022_SI_seq
, 1);
1925 if (ISO2022_MODE_CS(cv
->mode
) != ISO2022JP_CS_ASCII
)
1927 memcpy(buf
+ esc_len
, iesc
[ISO2022JP_CS_ASCII
].esc
,
1928 iesc
[ISO2022JP_CS_ASCII
].esc_len
);
1929 esc_len
+= iesc
[ISO2022JP_CS_ASCII
].esc_len
;
1936 #if defined(MAKE_DLL) && defined(USE_LIBICONV_DLL)
1938 DllMain(HINSTANCE hinstDLL
, DWORD fdwReason
, LPVOID lpReserved
)
1942 case DLL_PROCESS_ATTACH
:
1943 hwiniconv
= (HMODULE
)hinstDLL
;
1945 case DLL_THREAD_ATTACH
:
1946 case DLL_THREAD_DETACH
:
1947 case DLL_PROCESS_DETACH
:
1954 #if defined(MAKE_EXE)
1959 main(int argc
, char **argv
)
1961 char *fromcode
= NULL
;
1962 char *tocode
= NULL
;
1965 char outbuf
[BUFSIZ
];
1969 size_t outbytesleft
;
1978 _setmode(_fileno(stdin
), _O_BINARY
);
1979 _setmode(_fileno(stdout
), _O_BINARY
);
1981 for (i
= 1; i
< argc
; ++i
)
1983 if (strcmp(argv
[i
], "-l") == 0)
1985 for (i
= 0; codepage_alias
[i
].name
!= NULL
; ++i
)
1986 printf("%s\n", codepage_alias
[i
].name
);
1990 if (strcmp(argv
[i
], "-f") == 0)
1991 fromcode
= argv
[++i
];
1992 else if (strcmp(argv
[i
], "-t") == 0)
1994 else if (strcmp(argv
[i
], "-c") == 0)
1996 else if (strcmp(argv
[i
], "--output") == 0)
1998 out
= fopen(argv
[++i
], "wb");
2001 fprintf(stderr
, "cannot open %s\n", argv
[i
]);
2007 in
= fopen(argv
[i
], "rb");
2010 fprintf(stderr
, "cannot open %s\n", argv
[i
]);
2017 if (fromcode
== NULL
|| tocode
== NULL
)
2019 printf("usage: %s [-c] -f from-enc -t to-enc [file]\n", argv
[0]);
2026 tocode
= (char *)malloc(strlen(p
) + strlen("//IGNORE") + 1);
2029 perror("fatal error");
2033 strcat(tocode
, "//IGNORE");
2036 cd
= iconv_open(tocode
, fromcode
);
2037 if (cd
== (iconv_t
)(-1))
2039 perror("iconv_open error");
2043 while ((inbytesleft
= fread(inbuf
+ rest
, 1, sizeof(inbuf
) - rest
, in
)) != 0
2046 inbytesleft
+= rest
;
2049 outbytesleft
= sizeof(outbuf
);
2050 r
= iconv(cd
, &pin
, &inbytesleft
, &pout
, &outbytesleft
);
2051 fwrite(outbuf
, 1, sizeof(outbuf
) - outbytesleft
, out
);
2052 if (r
== (size_t)(-1) && errno
!= E2BIG
&& (errno
!= EINVAL
|| feof(in
)))
2054 perror("conversion error");
2057 memmove(inbuf
, pin
, inbytesleft
);
2061 outbytesleft
= sizeof(outbuf
);
2062 r
= iconv(cd
, NULL
, NULL
, &pout
, &outbytesleft
);
2063 fwrite(outbuf
, 1, sizeof(outbuf
) - outbytesleft
, out
);
2064 if (r
== (size_t)(-1))
2066 perror("conversion error");