1// © 2016 and later: Unicode, Inc. and others.
2// License & terms of use: http://www.unicode.org/copyright.html
3/*
4**********************************************************************
5* Copyright (C) 1999-2011, International Business Machines
6* Corporation and others. All Rights Reserved.
7**********************************************************************
8*
9* ucnv_cnv.h:
10* Definitions for converter implementations.
11*
12* Modification History:
13*
14* Date Name Description
15* 05/09/00 helena Added implementation to handle fallback mappings.
16* 06/29/2000 helena Major rewrite of the callback APIs.
17*/
18
19#ifndef UCNV_CNV_H
20#define UCNV_CNV_H
21
22#include "unicode/utypes.h"
23
24#if !UCONFIG_NO_CONVERSION
25
26#include "unicode/ucnv.h"
27#include "unicode/ucnv_err.h"
28#include "unicode/uset.h"
29#include "uset_imp.h"
30
31U_CDECL_BEGIN
32
33/* this is used in fromUnicode DBCS tables as an "unassigned" marker */
34#define missingCharMarker 0xFFFF
35
36/*
37 * #define missingUCharMarker 0xfffe
38 *
39 * commented out because there are actually two values used in toUnicode tables:
40 * U+fffe "unassigned"
41 * U+ffff "illegal"
42 */
43
44/** Forward declaration, see ucnv_bld.h */
45struct UConverterSharedData;
46typedef struct UConverterSharedData UConverterSharedData;
47
48/* function types for UConverterImpl ---------------------------------------- */
49
50/* struct with arguments for UConverterLoad and ucnv_load() */
51typedef struct {
52 int32_t size; /* sizeof(UConverterLoadArgs) */
53 int32_t nestedLoads; /* count nested ucnv_load() calls */
54 UBool onlyTestIsLoadable; /* input: don't actually load */
55 UBool reserved0; /* reserved - for good alignment of the pointers */
56 int16_t reserved; /* reserved - for good alignment of the pointers */
57 uint32_t options;
58 const char *pkg, *name, *locale;
59} UConverterLoadArgs;
60
61#define UCNV_LOAD_ARGS_INITIALIZER \
62 { (int32_t)sizeof(UConverterLoadArgs), 0, FALSE, FALSE, 0, 0, NULL, NULL, NULL }
63
64typedef void (*UConverterLoad) (UConverterSharedData *sharedData,
65 UConverterLoadArgs *pArgs,
66 const uint8_t *raw, UErrorCode *pErrorCode);
67typedef void (*UConverterUnload) (UConverterSharedData *sharedData);
68
69typedef void (*UConverterOpen) (UConverter *cnv, UConverterLoadArgs *pArgs, UErrorCode *pErrorCode);
70typedef void (*UConverterClose) (UConverter *cnv);
71
72typedef enum UConverterResetChoice {
73 UCNV_RESET_BOTH,
74 UCNV_RESET_TO_UNICODE,
75 UCNV_RESET_FROM_UNICODE
76} UConverterResetChoice;
77
78typedef void (*UConverterReset) (UConverter *cnv, UConverterResetChoice choice);
79
80/*
81 * Converter implementation function(s) for ucnv_toUnicode().
82 * If the toUnicodeWithOffsets function pointer is NULL,
83 * then the toUnicode function will be used and the offsets will be set to -1.
84 *
85 * Must maintain state across buffers. Use toUBytes[toULength] for partial input
86 * sequences; it will be checked in ucnv.c at the end of the input stream
87 * to detect truncated input.
88 * Some converters may need additional detection and may then set U_TRUNCATED_CHAR_FOUND.
89 *
90 * The toUnicodeWithOffsets must write exactly as many offset values as target
91 * units. Write offset values of -1 for when the source index corresponding to
92 * the output unit is not known (e.g., the character started in an earlier buffer).
93 * The pArgs->offsets pointer need not be moved forward.
94 *
95 * At function return, either one of the following conditions must be true:
96 * - U_BUFFER_OVERFLOW_ERROR and the target is full: target==targetLimit
97 * - another error code with toUBytes[toULength] set to the offending input
98 * - no error, and the source is consumed: source==sourceLimit
99 *
100 * The ucnv.c code will handle the end of the input (reset)
101 * (reset, and truncation detection) and callbacks.
102 */
103typedef void (*UConverterToUnicode) (UConverterToUnicodeArgs *, UErrorCode *);
104
105/*
106 * Same rules as for UConverterToUnicode.
107 * A lead surrogate is kept in fromUChar32 across buffers, and if an error
108 * occurs, then the offending input code point must be put into fromUChar32
109 * as well.
110 */
111typedef void (*UConverterFromUnicode) (UConverterFromUnicodeArgs *, UErrorCode *);
112
113/*
114 * Converter implementation function for ucnv_convertEx(), for direct conversion
115 * between two charsets without pivoting through UTF-16.
116 * The rules are the same as for UConverterToUnicode and UConverterFromUnicode.
117 * In addition,
118 * - The toUnicode side must behave and keep state exactly like the
119 * UConverterToUnicode implementation for the same source charset.
120 * - A U_USING_DEFAULT_WARNING can be set to request to temporarily fall back
121 * to pivoting. When this function is called, the conversion framework makes
122 * sure that this warning is not set on input.
123 * - Continuing a partial match and flushing the toUnicode replay buffer
124 * are handled by pivoting, using the toUnicode and fromUnicode functions.
125 */
126typedef void (*UConverterConvert) (UConverterFromUnicodeArgs *pFromUArgs,
127 UConverterToUnicodeArgs *pToUArgs,
128 UErrorCode *pErrorCode);
129
130/*
131 * Converter implementation function for ucnv_getNextUChar().
132 * If the function pointer is NULL, then the toUnicode function will be used.
133 *
134 * Will be called at a character boundary (toULength==0).
135 * May return with
136 * - U_INDEX_OUTOFBOUNDS_ERROR if there was no output for the input
137 * (the return value will be ignored)
138 * - U_TRUNCATED_CHAR_FOUND or another error code (never U_BUFFER_OVERFLOW_ERROR!)
139 * with toUBytes[toULength] set to the offending input
140 * (the return value will be ignored)
141 * - return UCNV_GET_NEXT_UCHAR_USE_TO_U, without moving the source pointer,
142 * to indicate that the ucnv.c code shall call the toUnicode function instead
143 * - return a real code point result
144 *
145 * Unless UCNV_GET_NEXT_UCHAR_USE_TO_U is returned, the source bytes must be consumed.
146 *
147 * The ucnv.c code will handle the end of the input (reset)
148 * (except for truncation detection!) and callbacks.
149 */
150typedef UChar32 (*UConverterGetNextUChar) (UConverterToUnicodeArgs *, UErrorCode *);
151
152typedef void (*UConverterGetStarters)(const UConverter* converter,
153 UBool starters[256],
154 UErrorCode *pErrorCode);
155
156/* If this function pointer is null or if the function returns null
157 * the name field in static data struct should be returned by
158 * ucnv_getName() API function
159 */
160typedef const char * (*UConverterGetName) (const UConverter *cnv);
161
162/**
163 * Write the codepage substitution character.
164 * If this function is not set, then ucnv_cbFromUWriteSub() writes
165 * the substitution character from UConverter.
166 * For stateful converters, it is typically necessary to handle this
167 * specificially for the converter in order to properly maintain the state.
168 */
169typedef void (*UConverterWriteSub) (UConverterFromUnicodeArgs *pArgs, int32_t offsetIndex, UErrorCode *pErrorCode);
170
171/**
172 * For converter-specific safeClone processing
173 * If this function is not set, then ucnv_safeClone assumes that the converter has no private data that changes
174 * after the converter is done opening.
175 * If this function is set, then it is called just after a memcpy() of
176 * converter data to the new, empty converter, and is expected to set up
177 * the initial state of the converter. It is not expected to increment the
178 * reference counts of the standard data types such as the shared data.
179 */
180typedef UConverter * (*UConverterSafeClone) (const UConverter *cnv,
181 void *stackBuffer,
182 int32_t *pBufferSize,
183 UErrorCode *status);
184
185/**
186 * Filters for some ucnv_getUnicodeSet() implementation code.
187 */
188typedef enum UConverterSetFilter {
189 UCNV_SET_FILTER_NONE,
190 UCNV_SET_FILTER_DBCS_ONLY,
191 UCNV_SET_FILTER_2022_CN,
192 UCNV_SET_FILTER_SJIS,
193 UCNV_SET_FILTER_GR94DBCS,
194 UCNV_SET_FILTER_HZ,
195 UCNV_SET_FILTER_COUNT
196} UConverterSetFilter;
197
198/**
199 * Fills the set of Unicode code points that can be converted by an ICU converter.
200 * The API function ucnv_getUnicodeSet() clears the USet before calling
201 * the converter's getUnicodeSet() implementation; the converter should only
202 * add the appropriate code points to allow recursive use.
203 * For example, the ISO-2022-JP converter will call each subconverter's
204 * getUnicodeSet() implementation to consecutively add code points to
205 * the same USet, which will result in a union of the sets of all subconverters.
206 *
207 * For more documentation, see ucnv_getUnicodeSet() in ucnv.h.
208 */
209typedef void (*UConverterGetUnicodeSet) (const UConverter *cnv,
210 const USetAdder *sa,
211 UConverterUnicodeSet which,
212 UErrorCode *pErrorCode);
213
214UBool CONVERSION_U_SUCCESS (UErrorCode err);
215
216/**
217 * UConverterImpl contains all the data and functions for a converter type.
218 * Its function pointers work much like a C++ vtable.
219 * Many converter types need to define only a subset of the functions;
220 * when a function pointer is NULL, then a default action will be performed.
221 *
222 * Every converter type must implement toUnicode, fromUnicode, and getNextUChar,
223 * otherwise the converter may crash.
224 * Every converter type that has variable-length codepage sequences should
225 * also implement toUnicodeWithOffsets and fromUnicodeWithOffsets for
226 * correct offset handling.
227 * All other functions may or may not be implemented - it depends only on
228 * whether the converter type needs them.
229 *
230 * When open() fails, then close() will be called, if present.
231 */
232struct UConverterImpl {
233 UConverterType type;
234
235 UConverterLoad load;
236 UConverterUnload unload;
237
238 UConverterOpen open;
239 UConverterClose close;
240 UConverterReset reset;
241
242 UConverterToUnicode toUnicode;
243 UConverterToUnicode toUnicodeWithOffsets;
244 UConverterFromUnicode fromUnicode;
245 UConverterFromUnicode fromUnicodeWithOffsets;
246 UConverterGetNextUChar getNextUChar;
247
248 UConverterGetStarters getStarters;
249 UConverterGetName getName;
250 UConverterWriteSub writeSub;
251 UConverterSafeClone safeClone;
252 UConverterGetUnicodeSet getUnicodeSet;
253
254 UConverterConvert toUTF8;
255 UConverterConvert fromUTF8;
256};
257
258extern const UConverterSharedData
259 _MBCSData, _Latin1Data,
260 _UTF8Data, _UTF16BEData, _UTF16LEData, _UTF32BEData, _UTF32LEData,
261 _ISO2022Data,
262 _LMBCSData1,_LMBCSData2, _LMBCSData3, _LMBCSData4, _LMBCSData5, _LMBCSData6,
263 _LMBCSData8,_LMBCSData11,_LMBCSData16,_LMBCSData17,_LMBCSData18,_LMBCSData19,
264 _HZData,_ISCIIData, _SCSUData, _ASCIIData,
265 _UTF7Data, _Bocu1Data, _UTF16Data, _UTF32Data, _CESU8Data, _IMAPData, _CompoundTextData;
266
267U_CDECL_END
268
269/** Always use fallbacks from codepage to Unicode */
270#define TO_U_USE_FALLBACK(useFallback) TRUE
271#define UCNV_TO_U_USE_FALLBACK(cnv) TRUE
272
273/** Use fallbacks from Unicode to codepage when cnv->useFallback or for private-use code points */
274#define IS_PRIVATE_USE(c) ((uint32_t)((c)-0xe000)<0x1900 || (uint32_t)((c)-0xf0000)<0x20000)
275#define FROM_U_USE_FALLBACK(useFallback, c) ((useFallback) || IS_PRIVATE_USE(c))
276#define UCNV_FROM_U_USE_FALLBACK(cnv, c) FROM_U_USE_FALLBACK((cnv)->useFallback, c)
277
278/**
279 * Magic number for ucnv_getNextUChar(), returned by a
280 * getNextUChar() implementation to indicate to use the converter's toUnicode()
281 * instead of the native function.
282 * @internal
283 */
284#define UCNV_GET_NEXT_UCHAR_USE_TO_U -9
285
286U_CFUNC void
287ucnv_getCompleteUnicodeSet(const UConverter *cnv,
288 const USetAdder *sa,
289 UConverterUnicodeSet which,
290 UErrorCode *pErrorCode);
291
292U_CFUNC void
293ucnv_getNonSurrogateUnicodeSet(const UConverter *cnv,
294 const USetAdder *sa,
295 UConverterUnicodeSet which,
296 UErrorCode *pErrorCode);
297
298U_CFUNC void
299ucnv_fromUWriteBytes(UConverter *cnv,
300 const char *bytes, int32_t length,
301 char **target, const char *targetLimit,
302 int32_t **offsets,
303 int32_t sourceIndex,
304 UErrorCode *pErrorCode);
305U_CFUNC void
306ucnv_toUWriteUChars(UConverter *cnv,
307 const UChar *uchars, int32_t length,
308 UChar **target, const UChar *targetLimit,
309 int32_t **offsets,
310 int32_t sourceIndex,
311 UErrorCode *pErrorCode);
312
313U_CFUNC void
314ucnv_toUWriteCodePoint(UConverter *cnv,
315 UChar32 c,
316 UChar **target, const UChar *targetLimit,
317 int32_t **offsets,
318 int32_t sourceIndex,
319 UErrorCode *pErrorCode);
320
321#endif
322
323#endif /* UCNV_CNV */
324