1// © 2016 and later: Unicode, Inc. and others.
2// License & terms of use: http://www.unicode.org/copyright.html
3/*
4*******************************************************************************
5* Copyright (c) 1996-2016, International Business Machines Corporation
6* and others. All Rights Reserved.
7*******************************************************************************
8* File unorm.h
9*
10* Created by: Vladimir Weinstein 12052000
11*
12* Modification history :
13*
14* Date Name Description
15* 02/01/01 synwee Added normalization quickcheck enum and method.
16*/
17#ifndef UNORM_H
18#define UNORM_H
19
20#include "unicode/utypes.h"
21
22#if !UCONFIG_NO_NORMALIZATION
23
24#include "unicode/uiter.h"
25#include "unicode/unorm2.h"
26
27/**
28 * \file
29 * \brief C API: Unicode Normalization
30 *
31 * Old Unicode normalization API.
32 *
33 * This API has been replaced by the unorm2.h API and is only available
34 * for backward compatibility. The functions here simply delegate to the
35 * unorm2.h functions, for example unorm2_getInstance() and unorm2_normalize().
36 * There is one exception: The new API does not provide a replacement for unorm_compare().
37 * Its declaration has been moved to unorm2.h.
38 *
39 * <code>unorm_normalize</code> transforms Unicode text into an equivalent composed or
40 * decomposed form, allowing for easier sorting and searching of text.
41 * <code>unorm_normalize</code> supports the standard normalization forms described in
42 * <a href="http://www.unicode.org/unicode/reports/tr15/" target="unicode">
43 * Unicode Standard Annex #15: Unicode Normalization Forms</a>.
44 *
45 * Characters with accents or other adornments can be encoded in
46 * several different ways in Unicode. For example, take the character A-acute.
47 * In Unicode, this can be encoded as a single character (the
48 * "composed" form):
49 *
50 * \code
51 * 00C1 LATIN CAPITAL LETTER A WITH ACUTE
52 * \endcode
53 *
54 * or as two separate characters (the "decomposed" form):
55 *
56 * \code
57 * 0041 LATIN CAPITAL LETTER A
58 * 0301 COMBINING ACUTE ACCENT
59 * \endcode
60 *
61 * To a user of your program, however, both of these sequences should be
62 * treated as the same "user-level" character "A with acute accent". When you are searching or
63 * comparing text, you must ensure that these two sequences are treated
64 * equivalently. In addition, you must handle characters with more than one
65 * accent. Sometimes the order of a character's combining accents is
66 * significant, while in other cases accent sequences in different orders are
67 * really equivalent.
68 *
69 * Similarly, the string "ffi" can be encoded as three separate letters:
70 *
71 * \code
72 * 0066 LATIN SMALL LETTER F
73 * 0066 LATIN SMALL LETTER F
74 * 0069 LATIN SMALL LETTER I
75 * \endcode
76 *
77 * or as the single character
78 *
79 * \code
80 * FB03 LATIN SMALL LIGATURE FFI
81 * \endcode
82 *
83 * The ffi ligature is not a distinct semantic character, and strictly speaking
84 * it shouldn't be in Unicode at all, but it was included for compatibility
85 * with existing character sets that already provided it. The Unicode standard
86 * identifies such characters by giving them "compatibility" decompositions
87 * into the corresponding semantic characters. When sorting and searching, you
88 * will often want to use these mappings.
89 *
90 * <code>unorm_normalize</code> helps solve these problems by transforming text into the
91 * canonical composed and decomposed forms as shown in the first example above.
92 * In addition, you can have it perform compatibility decompositions so that
93 * you can treat compatibility characters the same as their equivalents.
94 * Finally, <code>unorm_normalize</code> rearranges accents into the proper canonical
95 * order, so that you do not have to worry about accent rearrangement on your
96 * own.
97 *
98 * Form FCD, "Fast C or D", is also designed for collation.
99 * It allows to work on strings that are not necessarily normalized
100 * with an algorithm (like in collation) that works under "canonical closure", i.e., it treats precomposed
101 * characters and their decomposed equivalents the same.
102 *
103 * It is not a normalization form because it does not provide for uniqueness of representation. Multiple strings
104 * may be canonically equivalent (their NFDs are identical) and may all conform to FCD without being identical
105 * themselves.
106 *
107 * The form is defined such that the "raw decomposition", the recursive canonical decomposition of each character,
108 * results in a string that is canonically ordered. This means that precomposed characters are allowed for as long
109 * as their decompositions do not need canonical reordering.
110 *
111 * Its advantage for a process like collation is that all NFD and most NFC texts - and many unnormalized texts -
112 * already conform to FCD and do not need to be normalized (NFD) for such a process. The FCD quick check will
113 * return UNORM_YES for most strings in practice.
114 *
115 * unorm_normalize(UNORM_FCD) may be implemented with UNORM_NFD.
116 *
117 * For more details on FCD see the collation design document:
118 * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm
119 *
120 * ICU collation performs either NFD or FCD normalization automatically if normalization
121 * is turned on for the collator object.
122 * Beyond collation and string search, normalized strings may be useful for string equivalence comparisons,
123 * transliteration/transcription, unique representations, etc.
124 *
125 * The W3C generally recommends to exchange texts in NFC.
126 * Note also that most legacy character encodings use only precomposed forms and often do not
127 * encode any combining marks by themselves. For conversion to such character encodings the
128 * Unicode text needs to be normalized to NFC.
129 * For more usage examples, see the Unicode Standard Annex.
130 */
131
132// Do not conditionalize the following enum with #ifndef U_HIDE_DEPRECATED_API,
133// it is needed for layout of Normalizer object.
134/**
135 * Constants for normalization modes.
136 * @deprecated ICU 56 Use unorm2.h instead.
137 */
138typedef enum {
139 /** No decomposition/composition. @deprecated ICU 56 Use unorm2.h instead. */
140 UNORM_NONE = 1,
141 /** Canonical decomposition. @deprecated ICU 56 Use unorm2.h instead. */
142 UNORM_NFD = 2,
143 /** Compatibility decomposition. @deprecated ICU 56 Use unorm2.h instead. */
144 UNORM_NFKD = 3,
145 /** Canonical decomposition followed by canonical composition. @deprecated ICU 56 Use unorm2.h instead. */
146 UNORM_NFC = 4,
147 /** Default normalization. @deprecated ICU 56 Use unorm2.h instead. */
148 UNORM_DEFAULT = UNORM_NFC,
149 /** Compatibility decomposition followed by canonical composition. @deprecated ICU 56 Use unorm2.h instead. */
150 UNORM_NFKC =5,
151 /** "Fast C or D" form. @deprecated ICU 56 Use unorm2.h instead. */
152 UNORM_FCD = 6,
153
154 /** One more than the highest normalization mode constant. @deprecated ICU 56 Use unorm2.h instead. */
155 UNORM_MODE_COUNT
156} UNormalizationMode;
157
158#ifndef U_HIDE_DEPRECATED_API
159
160/**
161 * Constants for options flags for normalization.
162 * Use 0 for default options,
163 * including normalization according to the Unicode version
164 * that is currently supported by ICU (see u_getUnicodeVersion).
165 * @deprecated ICU 56 Use unorm2.h instead.
166 */
167enum {
168 /**
169 * Options bit set value to select Unicode 3.2 normalization
170 * (except NormalizationCorrections).
171 * At most one Unicode version can be selected at a time.
172 * @deprecated ICU 56 Use unorm2.h instead.
173 */
174 UNORM_UNICODE_3_2=0x20
175};
176
177/**
178 * Lowest-order bit number of unorm_compare() options bits corresponding to
179 * normalization options bits.
180 *
181 * The options parameter for unorm_compare() uses most bits for
182 * itself and for various comparison and folding flags.
183 * The most significant bits, however, are shifted down and passed on
184 * to the normalization implementation.
185 * (That is, from unorm_compare(..., options, ...),
186 * options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT will be passed on to the
187 * internal normalization functions.)
188 *
189 * @see unorm_compare
190 * @deprecated ICU 56 Use unorm2.h instead.
191 */
192#define UNORM_COMPARE_NORM_OPTIONS_SHIFT 20
193
194/**
195 * Normalize a string.
196 * The string will be normalized according the specified normalization mode
197 * and options.
198 * The source and result buffers must not be the same, nor overlap.
199 *
200 * @param source The string to normalize.
201 * @param sourceLength The length of source, or -1 if NUL-terminated.
202 * @param mode The normalization mode; one of UNORM_NONE,
203 * UNORM_NFD, UNORM_NFC, UNORM_NFKC, UNORM_NFKD, UNORM_DEFAULT.
204 * @param options The normalization options, ORed together (0 for no options).
205 * @param result A pointer to a buffer to receive the result string.
206 * The result string is NUL-terminated if possible.
207 * @param resultLength The maximum size of result.
208 * @param status A pointer to a UErrorCode to receive any errors.
209 * @return The total buffer size needed; if greater than resultLength,
210 * the output was truncated, and the error code is set to U_BUFFER_OVERFLOW_ERROR.
211 * @deprecated ICU 56 Use unorm2.h instead.
212 */
213U_DEPRECATED int32_t U_EXPORT2
214unorm_normalize(const UChar *source, int32_t sourceLength,
215 UNormalizationMode mode, int32_t options,
216 UChar *result, int32_t resultLength,
217 UErrorCode *status);
218
219/**
220 * Performing quick check on a string, to quickly determine if the string is
221 * in a particular normalization format.
222 * Three types of result can be returned UNORM_YES, UNORM_NO or
223 * UNORM_MAYBE. Result UNORM_YES indicates that the argument
224 * string is in the desired normalized format, UNORM_NO determines that
225 * argument string is not in the desired normalized format. A
226 * UNORM_MAYBE result indicates that a more thorough check is required,
227 * the user may have to put the string in its normalized form and compare the
228 * results.
229 *
230 * @param source string for determining if it is in a normalized format
231 * @param sourcelength length of source to test, or -1 if NUL-terminated
232 * @param mode which normalization form to test for
233 * @param status a pointer to a UErrorCode to receive any errors
234 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE
235 *
236 * @see unorm_isNormalized
237 * @deprecated ICU 56 Use unorm2.h instead.
238 */
239U_DEPRECATED UNormalizationCheckResult U_EXPORT2
240unorm_quickCheck(const UChar *source, int32_t sourcelength,
241 UNormalizationMode mode,
242 UErrorCode *status);
243
244/**
245 * Performing quick check on a string; same as unorm_quickCheck but
246 * takes an extra options parameter like most normalization functions.
247 *
248 * @param src String that is to be tested if it is in a normalization format.
249 * @param srcLength Length of source to test, or -1 if NUL-terminated.
250 * @param mode Which normalization form to test for.
251 * @param options The normalization options, ORed together (0 for no options).
252 * @param pErrorCode ICU error code in/out parameter.
253 * Must fulfill U_SUCCESS before the function call.
254 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE
255 *
256 * @see unorm_quickCheck
257 * @see unorm_isNormalized
258 * @deprecated ICU 56 Use unorm2.h instead.
259 */
260U_DEPRECATED UNormalizationCheckResult U_EXPORT2
261unorm_quickCheckWithOptions(const UChar *src, int32_t srcLength,
262 UNormalizationMode mode, int32_t options,
263 UErrorCode *pErrorCode);
264
265/**
266 * Test if a string is in a given normalization form.
267 * This is semantically equivalent to source.equals(normalize(source, mode)) .
268 *
269 * Unlike unorm_quickCheck(), this function returns a definitive result,
270 * never a "maybe".
271 * For NFD, NFKD, and FCD, both functions work exactly the same.
272 * For NFC and NFKC where quickCheck may return "maybe", this function will
273 * perform further tests to arrive at a TRUE/FALSE result.
274 *
275 * @param src String that is to be tested if it is in a normalization format.
276 * @param srcLength Length of source to test, or -1 if NUL-terminated.
277 * @param mode Which normalization form to test for.
278 * @param pErrorCode ICU error code in/out parameter.
279 * Must fulfill U_SUCCESS before the function call.
280 * @return Boolean value indicating whether the source string is in the
281 * "mode" normalization form.
282 *
283 * @see unorm_quickCheck
284 * @deprecated ICU 56 Use unorm2.h instead.
285 */
286U_DEPRECATED UBool U_EXPORT2
287unorm_isNormalized(const UChar *src, int32_t srcLength,
288 UNormalizationMode mode,
289 UErrorCode *pErrorCode);
290
291/**
292 * Test if a string is in a given normalization form; same as unorm_isNormalized but
293 * takes an extra options parameter like most normalization functions.
294 *
295 * @param src String that is to be tested if it is in a normalization format.
296 * @param srcLength Length of source to test, or -1 if NUL-terminated.
297 * @param mode Which normalization form to test for.
298 * @param options The normalization options, ORed together (0 for no options).
299 * @param pErrorCode ICU error code in/out parameter.
300 * Must fulfill U_SUCCESS before the function call.
301 * @return Boolean value indicating whether the source string is in the
302 * "mode/options" normalization form.
303 *
304 * @see unorm_quickCheck
305 * @see unorm_isNormalized
306 * @deprecated ICU 56 Use unorm2.h instead.
307 */
308U_DEPRECATED UBool U_EXPORT2
309unorm_isNormalizedWithOptions(const UChar *src, int32_t srcLength,
310 UNormalizationMode mode, int32_t options,
311 UErrorCode *pErrorCode);
312
313/**
314 * Iterative normalization forward.
315 * This function (together with unorm_previous) is somewhat
316 * similar to the C++ Normalizer class (see its non-static functions).
317 *
318 * Iterative normalization is useful when only a small portion of a longer
319 * string/text needs to be processed.
320 *
321 * For example, the likelihood may be high that processing the first 10% of some
322 * text will be sufficient to find certain data.
323 * Another example: When one wants to concatenate two normalized strings and get a
324 * normalized result, it is much more efficient to normalize just a small part of
325 * the result around the concatenation place instead of re-normalizing everything.
326 *
327 * The input text is an instance of the C character iteration API UCharIterator.
328 * It may wrap around a simple string, a CharacterIterator, a Replaceable, or any
329 * other kind of text object.
330 *
331 * If a buffer overflow occurs, then the caller needs to reset the iterator to the
332 * old index and call the function again with a larger buffer - if the caller cares
333 * for the actual output.
334 * Regardless of the output buffer, the iterator will always be moved to the next
335 * normalization boundary.
336 *
337 * This function (like unorm_previous) serves two purposes:
338 *
339 * 1) To find the next boundary so that the normalization of the part of the text
340 * from the current position to that boundary does not affect and is not affected
341 * by the part of the text beyond that boundary.
342 *
343 * 2) To normalize the text up to the boundary.
344 *
345 * The second step is optional, per the doNormalize parameter.
346 * It is omitted for operations like string concatenation, where the two adjacent
347 * string ends need to be normalized together.
348 * In such a case, the output buffer will just contain a copy of the text up to the
349 * boundary.
350 *
351 * pNeededToNormalize is an output-only parameter. Its output value is only defined
352 * if normalization was requested (doNormalize) and successful (especially, no
353 * buffer overflow).
354 * It is useful for operations like a normalizing transliterator, where one would
355 * not want to replace a piece of text if it is not modified.
356 *
357 * If doNormalize==TRUE and pNeededToNormalize!=NULL then *pNeeded... is set TRUE
358 * if the normalization was necessary.
359 *
360 * If doNormalize==FALSE then *pNeededToNormalize will be set to FALSE.
361 *
362 * If the buffer overflows, then *pNeededToNormalize will be undefined;
363 * essentially, whenever U_FAILURE is true (like in buffer overflows), this result
364 * will be undefined.
365 *
366 * @param src The input text in the form of a C character iterator.
367 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
368 * @param destCapacity The number of UChars that fit into dest.
369 * @param mode The normalization mode.
370 * @param options The normalization options, ORed together (0 for no options).
371 * @param doNormalize Indicates if the source text up to the next boundary
372 * is to be normalized (TRUE) or just copied (FALSE).
373 * @param pNeededToNormalize Output flag indicating if the normalization resulted in
374 * different text from the input.
375 * Not defined if an error occurs including buffer overflow.
376 * Always FALSE if !doNormalize.
377 * @param pErrorCode ICU error code in/out parameter.
378 * Must fulfill U_SUCCESS before the function call.
379 * @return Length of output (number of UChars) when successful or buffer overflow.
380 *
381 * @see unorm_previous
382 * @see unorm_normalize
383 *
384 * @deprecated ICU 56 Use unorm2.h instead.
385 */
386U_DEPRECATED int32_t U_EXPORT2
387unorm_next(UCharIterator *src,
388 UChar *dest, int32_t destCapacity,
389 UNormalizationMode mode, int32_t options,
390 UBool doNormalize, UBool *pNeededToNormalize,
391 UErrorCode *pErrorCode);
392
393/**
394 * Iterative normalization backward.
395 * This function (together with unorm_next) is somewhat
396 * similar to the C++ Normalizer class (see its non-static functions).
397 * For all details see unorm_next.
398 *
399 * @param src The input text in the form of a C character iterator.
400 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
401 * @param destCapacity The number of UChars that fit into dest.
402 * @param mode The normalization mode.
403 * @param options The normalization options, ORed together (0 for no options).
404 * @param doNormalize Indicates if the source text up to the next boundary
405 * is to be normalized (TRUE) or just copied (FALSE).
406 * @param pNeededToNormalize Output flag indicating if the normalization resulted in
407 * different text from the input.
408 * Not defined if an error occurs including buffer overflow.
409 * Always FALSE if !doNormalize.
410 * @param pErrorCode ICU error code in/out parameter.
411 * Must fulfill U_SUCCESS before the function call.
412 * @return Length of output (number of UChars) when successful or buffer overflow.
413 *
414 * @see unorm_next
415 * @see unorm_normalize
416 *
417 * @deprecated ICU 56 Use unorm2.h instead.
418 */
419U_DEPRECATED int32_t U_EXPORT2
420unorm_previous(UCharIterator *src,
421 UChar *dest, int32_t destCapacity,
422 UNormalizationMode mode, int32_t options,
423 UBool doNormalize, UBool *pNeededToNormalize,
424 UErrorCode *pErrorCode);
425
426/**
427 * Concatenate normalized strings, making sure that the result is normalized as well.
428 *
429 * If both the left and the right strings are in
430 * the normalization form according to "mode/options",
431 * then the result will be
432 *
433 * \code
434 * dest=normalize(left+right, mode, options)
435 * \endcode
436 *
437 * With the input strings already being normalized,
438 * this function will use unorm_next() and unorm_previous()
439 * to find the adjacent end pieces of the input strings.
440 * Only the concatenation of these end pieces will be normalized and
441 * then concatenated with the remaining parts of the input strings.
442 *
443 * It is allowed to have dest==left to avoid copying the entire left string.
444 *
445 * @param left Left source string, may be same as dest.
446 * @param leftLength Length of left source string, or -1 if NUL-terminated.
447 * @param right Right source string. Must not be the same as dest, nor overlap.
448 * @param rightLength Length of right source string, or -1 if NUL-terminated.
449 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
450 * @param destCapacity The number of UChars that fit into dest.
451 * @param mode The normalization mode.
452 * @param options The normalization options, ORed together (0 for no options).
453 * @param pErrorCode ICU error code in/out parameter.
454 * Must fulfill U_SUCCESS before the function call.
455 * @return Length of output (number of UChars) when successful or buffer overflow.
456 *
457 * @see unorm_normalize
458 * @see unorm_next
459 * @see unorm_previous
460 *
461 * @deprecated ICU 56 Use unorm2.h instead.
462 */
463U_DEPRECATED int32_t U_EXPORT2
464unorm_concatenate(const UChar *left, int32_t leftLength,
465 const UChar *right, int32_t rightLength,
466 UChar *dest, int32_t destCapacity,
467 UNormalizationMode mode, int32_t options,
468 UErrorCode *pErrorCode);
469
470#endif /* U_HIDE_DEPRECATED_API */
471#endif /* #if !UCONFIG_NO_NORMALIZATION */
472#endif
473