2 ******************************************************************************
4 * Copyright (C) 2003-2007, International Business Machines
5 * Corporation and others. All Rights Reserved.
7 ******************************************************************************
8 * file name: ucnv_ext.c
10 * tab size: 8 (not used)
13 * created on: 2003jun13
14 * created by: Markus W. Scherer
16 * Conversion extensions
19 #include "unicode/utypes.h"
21 #if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION
23 #include "unicode/uset.h"
29 /* to Unicode --------------------------------------------------------------- */
32 * @return lookup value for the byte, if found; else 0
34 static U_INLINE
uint32_t
35 ucnv_extFindToU(const uint32_t *toUSection
, int32_t length
, uint8_t byte
) {
37 int32_t i
, start
, limit
;
39 /* check the input byte against the lowest and highest section bytes */
40 start
=(int32_t)UCNV_EXT_TO_U_GET_BYTE(toUSection
[0]);
41 limit
=(int32_t)UCNV_EXT_TO_U_GET_BYTE(toUSection
[length
-1]);
42 if(byte
<start
|| limit
<byte
) {
43 return 0; /* the byte is out of range */
46 if(length
==((limit
-start
)+1)) {
47 /* direct access on a linear array */
48 return UCNV_EXT_TO_U_GET_VALUE(toUSection
[byte
-start
]); /* could be 0 */
51 /* word0 is suitable for <=toUSection[] comparison, word for <toUSection[] */
52 word0
=UCNV_EXT_TO_U_MAKE_WORD(byte
, 0);
55 * Shift byte once instead of each section word and add 0xffffff.
56 * We will compare the shifted/added byte (bbffffff) against
57 * section words which have byte values in the same bit position.
58 * If and only if byte bb < section byte ss then bbffffff<ssvvvvvv
60 * so we need not mask off the lower 24 bits of each section word.
62 word
=word0
|UCNV_EXT_TO_U_VALUE_MASK
;
75 /* linear search for the last part */
76 if(word0
<=toUSection
[start
]) {
79 if(++start
<limit
&& word0
<=toUSection
[start
]) {
82 if(++start
<limit
&& word0
<=toUSection
[start
]) {
85 /* always break at start==limit-1 */
91 if(word
<toUSection
[i
]) {
98 /* did we really find it? */
99 if(start
<limit
&& byte
==UCNV_EXT_TO_U_GET_BYTE(word
=toUSection
[start
])) {
100 return UCNV_EXT_TO_U_GET_VALUE(word
); /* never 0 */
102 return 0; /* not found */
107 * TRUE if not an SI/SO stateful converter,
108 * or if the match length fits with the current converter state
110 #define UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState, match) \
111 ((sisoState)<0 || ((sisoState)==0) == (match==1))
114 * this works like ucnv_extMatchFromU() except
115 * - the first character is in pre
117 * - the returned matchLength is not offset by 2
120 ucnv_extMatchToU(const int32_t *cx
, int8_t sisoState
,
121 const char *pre
, int32_t preLength
,
122 const char *src
, int32_t srcLength
,
123 uint32_t *pMatchValue
,
124 UBool useFallback
, UBool flush
) {
125 const uint32_t *toUTable
, *toUSection
;
127 uint32_t value
, matchValue
;
128 int32_t i
, j
, index
, length
, matchLength
;
131 if(cx
==NULL
|| cx
[UCNV_EXT_TO_U_LENGTH
]<=0) {
132 return 0; /* no extension data, no match */
136 toUTable
=UCNV_EXT_ARRAY(cx
, UCNV_EXT_TO_U_INDEX
, uint32_t);
143 /* SBCS state of an SI/SO stateful converter, look at only exactly 1 byte */
145 return 0; /* no match of a DBCS sequence in SBCS mode */
146 } else if(preLength
==1) {
148 } else /* preLength==0 */ {
156 /* we must not remember fallback matches when not using fallbacks */
158 /* match input units until there is a full match or the input is consumed */
160 /* go to the next section */
161 toUSection
=toUTable
+index
;
163 /* read first pair of the section */
165 length
=UCNV_EXT_TO_U_GET_BYTE(value
);
166 value
=UCNV_EXT_TO_U_GET_VALUE(value
);
168 (UCNV_EXT_TO_U_IS_ROUNDTRIP(value
) ||
169 TO_U_USE_FALLBACK(useFallback
)) &&
170 UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState
, i
+j
)
172 /* remember longest match so far */
177 /* match pre[] then src[] */
180 } else if(j
<srcLength
) {
183 /* all input consumed, partial match */
184 if(flush
|| (length
=(i
+j
))>UCNV_EXT_MAX_BYTES
) {
186 * end of the entire input stream, stop with the longest match so far
187 * or: partial match must not be longer than UCNV_EXT_MAX_BYTES
188 * because it must fit into state buffers
192 /* continue with more input next time */
197 /* search for the current UChar */
198 value
=ucnv_extFindToU(toUSection
, length
, b
);
200 /* no match here, stop with the longest match so far */
203 if(UCNV_EXT_TO_U_IS_PARTIAL(value
)) {
204 /* partial match, continue */
205 index
=(int32_t)UCNV_EXT_TO_U_GET_PARTIAL_INDEX(value
);
207 if( (UCNV_EXT_TO_U_IS_ROUNDTRIP(value
) ||
208 TO_U_USE_FALLBACK(useFallback
)) &&
209 UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState
, i
+j
)
211 /* full match, stop with result */
215 /* full match on fallback not taken, stop with the longest match so far */
223 /* no match at all */
228 *pMatchValue
=UCNV_EXT_TO_U_MASK_ROUNDTRIP(matchValue
);
233 ucnv_extWriteToU(UConverter
*cnv
, const int32_t *cx
,
235 UChar
**target
, const UChar
*targetLimit
,
236 int32_t **offsets
, int32_t srcIndex
,
237 UErrorCode
*pErrorCode
) {
238 /* output the result */
239 if(UCNV_EXT_TO_U_IS_CODE_POINT(value
)) {
240 /* output a single code point */
241 ucnv_toUWriteCodePoint(
242 cnv
, UCNV_EXT_TO_U_GET_CODE_POINT(value
),
247 /* output a string - with correct data we have resultLength>0 */
250 UCNV_EXT_ARRAY(cx
, UCNV_EXT_TO_U_UCHARS_INDEX
, UChar
)+
251 UCNV_EXT_TO_U_GET_INDEX(value
),
252 UCNV_EXT_TO_U_GET_LENGTH(value
),
260 * get the SI/SO toU state (state 0 is for SBCS, 1 for DBCS),
261 * or 1 for DBCS-only,
262 * or -1 if the converter is not SI/SO stateful
264 * Note: For SI/SO stateful converters getting here,
265 * cnv->mode==0 is equivalent to firstLength==1.
267 #define UCNV_SISO_STATE(cnv) \
268 ((cnv)->sharedData->mbcs.outputType==MBCS_OUTPUT_2_SISO ? (int8_t)(cnv)->mode : \
269 (cnv)->sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY ? 1 : -1)
272 * target<targetLimit; set error code for overflow
275 ucnv_extInitialMatchToU(UConverter
*cnv
, const int32_t *cx
,
277 const char **src
, const char *srcLimit
,
278 UChar
**target
, const UChar
*targetLimit
,
279 int32_t **offsets
, int32_t srcIndex
,
281 UErrorCode
*pErrorCode
) {
286 match
=ucnv_extMatchToU(cx
, (int8_t)UCNV_SISO_STATE(cnv
),
287 (const char *)cnv
->toUBytes
, firstLength
,
288 *src
, (int32_t)(srcLimit
-*src
),
290 cnv
->useFallback
, flush
);
292 /* advance src pointer for the consumed input */
293 *src
+=match
-firstLength
;
295 /* write result to target */
296 ucnv_extWriteToU(cnv
, cx
,
303 /* save state for partial match */
307 /* copy the first code point */
308 s
=(const char *)cnv
->toUBytes
;
309 cnv
->preToUFirstLength
=(int8_t)firstLength
;
310 for(j
=0; j
<firstLength
; ++j
) {
314 /* now copy the newly consumed input */
317 for(; j
<match
; ++j
) {
320 *src
=s
; /* same as *src=srcLimit; because we reached the end of input */
321 cnv
->preToULength
=(int8_t)match
;
323 } else /* match==0 no match */ {
329 ucnv_extSimpleMatchToU(const int32_t *cx
,
330 const char *source
, int32_t length
,
340 match
=ucnv_extMatchToU(cx
, -1,
346 /* write result for simple, single-character conversion */
347 if(UCNV_EXT_TO_U_IS_CODE_POINT(value
)) {
348 return UCNV_EXT_TO_U_GET_CODE_POINT(value
);
353 * return no match because
354 * - match>0 && value points to string: simple conversion cannot handle multiple code points
355 * - match>0 && match!=length: not all input consumed, forbidden for this function
356 * - match==0: no match found in the first place
357 * - match<0: partial match, not supported for simple conversion (and flush==TRUE)
363 * continue partial match with new input
364 * never called for simple, single-character conversion
367 ucnv_extContinueMatchToU(UConverter
*cnv
,
368 UConverterToUnicodeArgs
*pArgs
, int32_t srcIndex
,
369 UErrorCode
*pErrorCode
) {
371 int32_t match
, length
;
373 match
=ucnv_extMatchToU(cnv
->sharedData
->mbcs
.extIndexes
, (int8_t)UCNV_SISO_STATE(cnv
),
374 cnv
->preToU
, cnv
->preToULength
,
375 pArgs
->source
, (int32_t)(pArgs
->sourceLimit
-pArgs
->source
),
377 cnv
->useFallback
, pArgs
->flush
);
379 if(match
>=cnv
->preToULength
) {
380 /* advance src pointer for the consumed input */
381 pArgs
->source
+=match
-cnv
->preToULength
;
384 /* the match did not use all of preToU[] - keep the rest for replay */
385 length
=cnv
->preToULength
-match
;
386 uprv_memmove(cnv
->preToU
, cnv
->preToU
+match
, length
);
387 cnv
->preToULength
=(int8_t)-length
;
391 ucnv_extWriteToU(cnv
, cnv
->sharedData
->mbcs
.extIndexes
,
393 &pArgs
->target
, pArgs
->targetLimit
,
394 &pArgs
->offsets
, srcIndex
,
397 /* save state for partial match */
401 /* just _append_ the newly consumed input to preToU[] */
404 for(j
=cnv
->preToULength
; j
<match
; ++j
) {
407 pArgs
->source
=s
; /* same as *src=srcLimit; because we reached the end of input */
408 cnv
->preToULength
=(int8_t)match
;
409 } else /* match==0 */ {
413 * We need to split the previous input into two parts:
415 * 1. The first codepage character is unmappable - that's how we got into
416 * trying the extension data in the first place.
417 * We need to move it from the preToU buffer
418 * to the error buffer, set an error code,
419 * and prepare the rest of the previous input for 2.
421 * 2. The rest of the previous input must be converted once we
422 * come back from the callback for the first character.
423 * At that time, we have to try again from scratch to convert
424 * these input characters.
425 * The replay will be handled by the ucnv.c conversion code.
428 /* move the first codepage character to the error field */
429 uprv_memcpy(cnv
->toUBytes
, cnv
->preToU
, cnv
->preToUFirstLength
);
430 cnv
->toULength
=cnv
->preToUFirstLength
;
432 /* move the rest up inside the buffer */
433 length
=cnv
->preToULength
-cnv
->preToUFirstLength
;
435 uprv_memmove(cnv
->preToU
, cnv
->preToU
+cnv
->preToUFirstLength
, length
);
438 /* mark preToU for replay */
439 cnv
->preToULength
=(int8_t)-length
;
441 /* set the error code for unassigned */
442 *pErrorCode
=U_INVALID_CHAR_FOUND
;
446 /* from Unicode ------------------------------------------------------------- */
449 * @return index of the UChar, if found; else <0
451 static U_INLINE
int32_t
452 ucnv_extFindFromU(const UChar
*fromUSection
, int32_t length
, UChar u
) {
453 int32_t i
, start
, limit
;
466 /* linear search for the last part */
467 if(u
<=fromUSection
[start
]) {
470 if(++start
<limit
&& u
<=fromUSection
[start
]) {
473 if(++start
<limit
&& u
<=fromUSection
[start
]) {
476 /* always break at start==limit-1 */
482 if(u
<fromUSection
[i
]) {
489 /* did we really find it? */
490 if(start
<limit
&& u
==fromUSection
[start
]) {
493 return -1; /* not found */
498 * @param cx pointer to extension data; if NULL, returns 0
499 * @param firstCP the first code point before all the other UChars
500 * @param pre UChars that must match; !initialMatch: partial match with them
501 * @param preLength length of pre, >=0
502 * @param src UChars that can be used to complete a match
503 * @param srcLength length of src, >=0
504 * @param pMatchValue [out] output result value for the match from the data structure
505 * @param useFallback "use fallback" flag, usually from cnv->useFallback
506 * @param flush TRUE if the end of the input stream is reached
507 * @return >1: matched, return value=total match length (number of input units matched)
508 * 1: matched, no mapping but request for <subchar1>
509 * (only for the first code point)
511 * <0: partial match, return value=negative total match length
512 * (partial matches are never returned for flush==TRUE)
513 * (partial matches are never returned as being longer than UCNV_EXT_MAX_UCHARS)
514 * the matchLength is 2 if only firstCP matched, and >2 if firstCP and
515 * further code units matched
518 ucnv_extMatchFromU(const int32_t *cx
,
520 const UChar
*pre
, int32_t preLength
,
521 const UChar
*src
, int32_t srcLength
,
522 uint32_t *pMatchValue
,
523 UBool useFallback
, UBool flush
) {
524 const uint16_t *stage12
, *stage3
;
525 const uint32_t *stage3b
;
527 const UChar
*fromUTableUChars
, *fromUSectionUChars
;
528 const uint32_t *fromUTableValues
, *fromUSectionValues
;
530 uint32_t value
, matchValue
;
531 int32_t i
, j
, index
, length
, matchLength
;
535 return 0; /* no extension data, no match */
538 /* trie lookup of firstCP */
539 index
=firstCP
>>10; /* stage 1 index */
540 if(index
>=cx
[UCNV_EXT_FROM_U_STAGE_1_LENGTH
]) {
541 return 0; /* the first code point is outside the trie */
544 stage12
=UCNV_EXT_ARRAY(cx
, UCNV_EXT_FROM_U_STAGE_12_INDEX
, uint16_t);
545 stage3
=UCNV_EXT_ARRAY(cx
, UCNV_EXT_FROM_U_STAGE_3_INDEX
, uint16_t);
546 index
=UCNV_EXT_FROM_U(stage12
, stage3
, index
, firstCP
);
548 stage3b
=UCNV_EXT_ARRAY(cx
, UCNV_EXT_FROM_U_STAGE_3B_INDEX
, uint32_t);
549 value
=stage3b
[index
];
555 * Tests for (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0:
556 * Do not interpret values with reserved bits used, for forward compatibility,
557 * and do not even remember intermediate results with reserved bits used.
560 if(UCNV_EXT_TO_U_IS_PARTIAL(value
)) {
561 /* partial match, enter the loop below */
562 index
=(int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value
);
565 fromUTableUChars
=UCNV_EXT_ARRAY(cx
, UCNV_EXT_FROM_U_UCHARS_INDEX
, UChar
);
566 fromUTableValues
=UCNV_EXT_ARRAY(cx
, UCNV_EXT_FROM_U_VALUES_INDEX
, uint32_t);
571 /* we must not remember fallback matches when not using fallbacks */
573 /* match input units until there is a full match or the input is consumed */
575 /* go to the next section */
576 fromUSectionUChars
=fromUTableUChars
+index
;
577 fromUSectionValues
=fromUTableValues
+index
;
579 /* read first pair of the section */
580 length
=*fromUSectionUChars
++;
581 value
=*fromUSectionValues
++;
583 (UCNV_EXT_FROM_U_IS_ROUNDTRIP(value
) ||
584 FROM_U_USE_FALLBACK(useFallback
, firstCP
)) &&
585 (value
&UCNV_EXT_FROM_U_RESERVED_MASK
)==0
587 /* remember longest match so far */
592 /* match pre[] then src[] */
595 } else if(j
<srcLength
) {
598 /* all input consumed, partial match */
599 if(flush
|| (length
=(i
+j
))>UCNV_EXT_MAX_UCHARS
) {
601 * end of the entire input stream, stop with the longest match so far
602 * or: partial match must not be longer than UCNV_EXT_MAX_UCHARS
603 * because it must fit into state buffers
607 /* continue with more input next time */
612 /* search for the current UChar */
613 index
=ucnv_extFindFromU(fromUSectionUChars
, length
, c
);
615 /* no match here, stop with the longest match so far */
618 value
=fromUSectionValues
[index
];
619 if(UCNV_EXT_FROM_U_IS_PARTIAL(value
)) {
620 /* partial match, continue */
621 index
=(int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value
);
623 if( (UCNV_EXT_FROM_U_IS_ROUNDTRIP(value
) ||
624 FROM_U_USE_FALLBACK(useFallback
, firstCP
)) &&
625 (value
&UCNV_EXT_FROM_U_RESERVED_MASK
)==0
627 /* full match, stop with result */
631 /* full match on fallback not taken, stop with the longest match so far */
639 /* no match at all */
642 } else /* result from firstCP trie lookup */ {
643 if( (UCNV_EXT_FROM_U_IS_ROUNDTRIP(value
) ||
644 FROM_U_USE_FALLBACK(useFallback
, firstCP
)) &&
645 (value
&UCNV_EXT_FROM_U_RESERVED_MASK
)==0
647 /* full match, stop with result */
651 /* fallback not taken */
657 if(matchValue
==UCNV_EXT_FROM_U_SUBCHAR1
) {
658 return 1; /* assert matchLength==2 */
661 *pMatchValue
=matchValue
;
666 * @param value fromUnicode mapping table value; ignores roundtrip and reserved bits
669 ucnv_extWriteFromU(UConverter
*cnv
, const int32_t *cx
,
671 char **target
, const char *targetLimit
,
672 int32_t **offsets
, int32_t srcIndex
,
673 UErrorCode
*pErrorCode
) {
674 uint8_t buffer
[1+UCNV_EXT_MAX_BYTES
];
675 const uint8_t *result
;
676 int32_t length
, prevLength
;
678 length
=UCNV_EXT_FROM_U_GET_LENGTH(value
);
679 value
=(uint32_t)UCNV_EXT_FROM_U_GET_DATA(value
);
681 /* output the result */
682 if(length
<=UCNV_EXT_FROM_U_MAX_DIRECT_LENGTH
) {
684 * Generate a byte array and then write it below.
685 * This is not the fastest possible way, but it should be ok for
686 * extension mappings, and it is much simpler.
687 * Offset and overflow handling are only done once this way.
689 uint8_t *p
=buffer
+1; /* reserve buffer[0] for shiftByte below */
692 *p
++=(uint8_t)(value
>>16);
694 *p
++=(uint8_t)(value
>>8);
698 break; /* will never occur */
702 result
=UCNV_EXT_ARRAY(cx
, UCNV_EXT_FROM_U_BYTES_INDEX
, uint8_t)+value
;
705 /* with correct data we have length>0 */
707 if((prevLength
=cnv
->fromUnicodeStatus
)!=0) {
708 /* handle SI/SO stateful output */
711 if(prevLength
>1 && length
==1) {
712 /* change from double-byte mode to single-byte */
713 shiftByte
=(uint8_t)UCNV_SI
;
714 cnv
->fromUnicodeStatus
=1;
715 } else if(prevLength
==1 && length
>1) {
716 /* change from single-byte mode to double-byte */
717 shiftByte
=(uint8_t)UCNV_SO
;
718 cnv
->fromUnicodeStatus
=2;
724 /* prepend the shift byte to the result bytes */
726 if(result
!=buffer
+1) {
727 uprv_memcpy(buffer
+1, result
, length
);
734 ucnv_fromUWriteBytes(cnv
, (const char *)result
, length
,
741 * target<targetLimit; set error code for overflow
744 ucnv_extInitialMatchFromU(UConverter
*cnv
, const int32_t *cx
,
746 const UChar
**src
, const UChar
*srcLimit
,
747 char **target
, const char *targetLimit
,
748 int32_t **offsets
, int32_t srcIndex
,
750 UErrorCode
*pErrorCode
) {
755 match
=ucnv_extMatchFromU(cx
, cp
,
757 *src
, (int32_t)(srcLimit
-*src
),
759 cnv
->useFallback
, flush
);
761 /* reject a match if the result is a single byte for DBCS-only */
763 !(UCNV_EXT_FROM_U_GET_LENGTH(value
)==1 &&
764 cnv
->sharedData
->mbcs
.outputType
==MBCS_OUTPUT_DBCS_ONLY
)
766 /* advance src pointer for the consumed input */
767 *src
+=match
-2; /* remove 2 for the initial code point */
769 /* write result to target */
770 ucnv_extWriteFromU(cnv
, cx
,
777 /* save state for partial match */
781 /* copy the first code point */
782 cnv
->preFromUFirstCP
=cp
;
784 /* now copy the newly consumed input */
786 match
=-match
-2; /* remove 2 for the initial code point */
787 for(j
=0; j
<match
; ++j
) {
788 cnv
->preFromU
[j
]=*s
++;
790 *src
=s
; /* same as *src=srcLimit; because we reached the end of input */
791 cnv
->preFromULength
=(int8_t)match
;
793 } else if(match
==1) {
794 /* matched, no mapping but request for <subchar1> */
795 cnv
->useSubChar1
=TRUE
;
797 } else /* match==0 no match */ {
803 * Used by ISO 2022 implementation.
804 * @return number of bytes in *pValue; negative number if fallback; 0 for no mapping
807 ucnv_extSimpleMatchFromU(const int32_t *cx
,
808 UChar32 cp
, uint32_t *pValue
,
814 match
=ucnv_extMatchFromU(cx
,
821 /* write result for simple, single-character conversion */
825 isRoundtrip
=UCNV_EXT_FROM_U_IS_ROUNDTRIP(value
);
826 length
=UCNV_EXT_FROM_U_GET_LENGTH(value
);
827 value
=(uint32_t)UCNV_EXT_FROM_U_GET_DATA(value
);
829 if(length
<=UCNV_EXT_FROM_U_MAX_DIRECT_LENGTH
) {
831 return isRoundtrip
? length
: -length
;
832 #if 0 /* not currently used */
833 } else if(length
==4) {
834 /* de-serialize a 4-byte result */
835 const uint8_t *result
=UCNV_EXT_ARRAY(cx
, UCNV_EXT_FROM_U_BYTES_INDEX
, uint8_t)+value
;
837 ((uint32_t)result
[0]<<24)|
838 ((uint32_t)result
[1]<<16)|
839 ((uint32_t)result
[2]<<8)|
841 return isRoundtrip
? 4 : -4;
847 * return no match because
848 * - match>1 && resultLength>4: result too long for simple conversion
849 * - match==1: no match found, <subchar1> preferred
850 * - match==0: no match found in the first place
851 * - match<0: partial match, not supported for simple conversion (and flush==TRUE)
857 * continue partial match with new input, requires cnv->preFromUFirstCP>=0
858 * never called for simple, single-character conversion
861 ucnv_extContinueMatchFromU(UConverter
*cnv
,
862 UConverterFromUnicodeArgs
*pArgs
, int32_t srcIndex
,
863 UErrorCode
*pErrorCode
) {
867 match
=ucnv_extMatchFromU(cnv
->sharedData
->mbcs
.extIndexes
,
868 cnv
->preFromUFirstCP
,
869 cnv
->preFromU
, cnv
->preFromULength
,
870 pArgs
->source
, (int32_t)(pArgs
->sourceLimit
-pArgs
->source
),
872 cnv
->useFallback
, pArgs
->flush
);
874 match
-=2; /* remove 2 for the initial code point */
876 if(match
>=cnv
->preFromULength
) {
877 /* advance src pointer for the consumed input */
878 pArgs
->source
+=match
-cnv
->preFromULength
;
879 cnv
->preFromULength
=0;
881 /* the match did not use all of preFromU[] - keep the rest for replay */
882 int32_t length
=cnv
->preFromULength
-match
;
883 uprv_memmove(cnv
->preFromU
, cnv
->preFromU
+match
, length
*U_SIZEOF_UCHAR
);
884 cnv
->preFromULength
=(int8_t)-length
;
887 /* finish the partial match */
888 cnv
->preFromUFirstCP
=U_SENTINEL
;
891 ucnv_extWriteFromU(cnv
, cnv
->sharedData
->mbcs
.extIndexes
,
893 &pArgs
->target
, pArgs
->targetLimit
,
894 &pArgs
->offsets
, srcIndex
,
897 /* save state for partial match */
901 /* just _append_ the newly consumed input to preFromU[] */
903 match
=-match
-2; /* remove 2 for the initial code point */
904 for(j
=cnv
->preFromULength
; j
<match
; ++j
) {
905 cnv
->preFromU
[j
]=*s
++;
907 pArgs
->source
=s
; /* same as *src=srcLimit; because we reached the end of input */
908 cnv
->preFromULength
=(int8_t)match
;
909 } else /* match==0 or 1 */ {
913 * We need to split the previous input into two parts:
915 * 1. The first code point is unmappable - that's how we got into
916 * trying the extension data in the first place.
917 * We need to move it from the preFromU buffer
918 * to the error buffer, set an error code,
919 * and prepare the rest of the previous input for 2.
921 * 2. The rest of the previous input must be converted once we
922 * come back from the callback for the first code point.
923 * At that time, we have to try again from scratch to convert
924 * these input characters.
925 * The replay will be handled by the ucnv.c conversion code.
929 /* matched, no mapping but request for <subchar1> */
930 cnv
->useSubChar1
=TRUE
;
933 /* move the first code point to the error field */
934 cnv
->fromUChar32
=cnv
->preFromUFirstCP
;
935 cnv
->preFromUFirstCP
=U_SENTINEL
;
937 /* mark preFromU for replay */
938 cnv
->preFromULength
=-cnv
->preFromULength
;
940 /* set the error code for unassigned */
941 *pErrorCode
=U_INVALID_CHAR_FOUND
;
946 ucnv_extGetUnicodeSetString(const UConverterSharedData
*sharedData
,
949 UConverterUnicodeSet which
,
952 UChar s
[UCNV_EXT_MAX_UCHARS
], int32_t length
,
953 int32_t sectionIndex
,
954 UErrorCode
*pErrorCode
) {
955 const UChar
*fromUSectionUChars
;
956 const uint32_t *fromUSectionValues
;
961 fromUSectionUChars
=UCNV_EXT_ARRAY(cx
, UCNV_EXT_FROM_U_UCHARS_INDEX
, UChar
)+sectionIndex
;
962 fromUSectionValues
=UCNV_EXT_ARRAY(cx
, UCNV_EXT_FROM_U_VALUES_INDEX
, uint32_t)+sectionIndex
;
964 /* read first pair of the section */
965 count
=*fromUSectionUChars
++;
966 value
=*fromUSectionValues
++;
969 UCNV_EXT_FROM_U_IS_ROUNDTRIP(value
) &&
970 UCNV_EXT_FROM_U_GET_LENGTH(value
)>=minLength
973 /* add the initial code point */
976 /* add the string so far */
977 sa
->addString(sa
->set
, s
, length
);
981 for(i
=0; i
<count
; ++i
) {
982 /* append this code unit and recurse or add the string */
983 s
[length
]=fromUSectionUChars
[i
];
984 value
=fromUSectionValues
[i
];
987 /* no mapping, do nothing */
988 } else if(UCNV_EXT_FROM_U_IS_PARTIAL(value
)) {
989 ucnv_extGetUnicodeSetString(
990 sharedData
, cx
, sa
, which
, minLength
,
991 U_SENTINEL
, s
, length
+1,
992 (int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value
),
994 } else if(((value
&(UCNV_EXT_FROM_U_ROUNDTRIP_FLAG
|UCNV_EXT_FROM_U_RESERVED_MASK
))==
995 UCNV_EXT_FROM_U_ROUNDTRIP_FLAG
) &&
996 UCNV_EXT_FROM_U_GET_LENGTH(value
)>=minLength
998 sa
->addString(sa
->set
, s
, length
+1);
1004 ucnv_extGetUnicodeSet(const UConverterSharedData
*sharedData
,
1005 const USetAdder
*sa
,
1006 UConverterUnicodeSet which
,
1007 UErrorCode
*pErrorCode
) {
1009 const uint16_t *stage12
, *stage3
, *ps2
, *ps3
;
1010 const uint32_t *stage3b
;
1013 int32_t st1
, stage1Length
, st2
, st3
, minLength
;
1015 UChar s
[UCNV_EXT_MAX_UCHARS
];
1019 cx
=sharedData
->mbcs
.extIndexes
;
1024 stage12
=UCNV_EXT_ARRAY(cx
, UCNV_EXT_FROM_U_STAGE_12_INDEX
, uint16_t);
1025 stage3
=UCNV_EXT_ARRAY(cx
, UCNV_EXT_FROM_U_STAGE_3_INDEX
, uint16_t);
1026 stage3b
=UCNV_EXT_ARRAY(cx
, UCNV_EXT_FROM_U_STAGE_3B_INDEX
, uint32_t);
1028 stage1Length
=cx
[UCNV_EXT_FROM_U_STAGE_1_LENGTH
];
1030 /* enumerate the from-Unicode trie table */
1031 c
=0; /* keep track of the current code point while enumerating */
1033 if(sharedData
->mbcs
.outputType
==MBCS_OUTPUT_DBCS_ONLY
) {
1034 /* DBCS-only, ignore single-byte results */
1041 * the trie enumeration is almost the same as
1042 * in MBCSGetUnicodeSet() for MBCS_OUTPUT_1
1044 for(st1
=0; st1
<stage1Length
; ++st1
) {
1046 if(st2
>stage1Length
) {
1048 for(st2
=0; st2
<64; ++st2
) {
1049 if((st3
=(int32_t)ps2
[st2
]<<UCNV_EXT_STAGE_2_LEFT_SHIFT
)!=0) {
1050 /* read the stage 3 block */
1054 * Add code points for which the roundtrip flag is set.
1055 * Do not add <subchar1> entries or other (future?) pseudo-entries
1056 * with an output length of 0, or entries with reserved bits set.
1057 * Recurse for partial results.
1060 value
=stage3b
[*ps3
++];
1062 /* no mapping, do nothing */
1063 } else if(UCNV_EXT_FROM_U_IS_PARTIAL(value
)) {
1065 U16_APPEND_UNSAFE(s
, length
, c
);
1066 ucnv_extGetUnicodeSetString(
1067 sharedData
, cx
, sa
, which
, minLength
,
1069 (int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value
),
1071 } else if(((value
&(UCNV_EXT_FROM_U_ROUNDTRIP_FLAG
|UCNV_EXT_FROM_U_RESERVED_MASK
))==
1072 UCNV_EXT_FROM_U_ROUNDTRIP_FLAG
) &&
1073 UCNV_EXT_FROM_U_GET_LENGTH(value
)>=minLength
1075 sa
->add(sa
->set
, c
);
1077 } while((++c
&0xf)!=0);
1079 c
+=16; /* empty stage 3 block */
1083 c
+=1024; /* empty stage 2 block */
1088 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */