1// © 2016 and later: Unicode, Inc. and others.
2// License & terms of use: http://www.unicode.org/copyright.html
3/*
4*******************************************************************************
5* Copyright (C) 2010-2015, International Business Machines
6* Corporation and others. All Rights Reserved.
7*******************************************************************************
8* collationdata.h
9*
10* created on: 2010oct27
11* created by: Markus W. Scherer
12*/
13
14#ifndef __COLLATIONDATA_H__
15#define __COLLATIONDATA_H__
16
17#include "unicode/utypes.h"
18
19#if !UCONFIG_NO_COLLATION
20
21#include "unicode/ucol.h"
22#include "unicode/uniset.h"
23#include "collation.h"
24#include "normalizer2impl.h"
25#include "utrie2.h"
26
27struct UDataMemory;
28
29U_NAMESPACE_BEGIN
30
31class UVector32;
32
33/**
34 * Collation data container.
35 * Immutable data created by a CollationDataBuilder, or loaded from a file,
36 * or deserialized from API-provided binary data.
37 *
38 * Includes data for the collation base (root/default), aliased if this is not the base.
39 */
40struct U_I18N_API CollationData : public UMemory {
41 // Note: The ucadata.icu loader could discover the reserved ranges by setting an array
42 // parallel with the ranges, and resetting ranges that are indexed.
43 // The reordering builder code could clone the resulting template array.
44 enum {
45 REORDER_RESERVED_BEFORE_LATIN = UCOL_REORDER_CODE_FIRST + 14,
46 REORDER_RESERVED_AFTER_LATIN
47 };
48
49 enum {
50 MAX_NUM_SPECIAL_REORDER_CODES = 8,
51 /** C++ only, data reader check scriptStartsLength. */
52 MAX_NUM_SCRIPT_RANGES = 256
53 };
54
55 CollationData(const Normalizer2Impl &nfc)
56 : trie(NULL),
57 ce32s(NULL), ces(NULL), contexts(NULL), base(NULL),
58 jamoCE32s(NULL),
59 nfcImpl(nfc),
60 numericPrimary(0x12000000),
61 ce32sLength(0), cesLength(0), contextsLength(0),
62 compressibleBytes(NULL),
63 unsafeBackwardSet(NULL),
64 fastLatinTable(NULL), fastLatinTableLength(0),
65 numScripts(0), scriptsIndex(NULL), scriptStarts(NULL), scriptStartsLength(0),
66 rootElements(NULL), rootElementsLength(0) {}
67
68 uint32_t getCE32(UChar32 c) const {
69 return UTRIE2_GET32(trie, c);
70 }
71
72 uint32_t getCE32FromSupplementary(UChar32 c) const {
73 return UTRIE2_GET32_FROM_SUPP(trie, c);
74 }
75
76 UBool isDigit(UChar32 c) const {
77 return c < 0x660 ? c <= 0x39 && 0x30 <= c :
78 Collation::hasCE32Tag(getCE32(c), Collation::DIGIT_TAG);
79 }
80
81 UBool isUnsafeBackward(UChar32 c, UBool numeric) const {
82 return unsafeBackwardSet->contains(c) || (numeric && isDigit(c));
83 }
84
85 UBool isCompressibleLeadByte(uint32_t b) const {
86 return compressibleBytes[b];
87 }
88
89 inline UBool isCompressiblePrimary(uint32_t p) const {
90 return isCompressibleLeadByte(p >> 24);
91 }
92
93 /**
94 * Returns the CE32 from two contexts words.
95 * Access to the defaultCE32 for contraction and prefix matching.
96 */
97 static uint32_t readCE32(const UChar *p) {
98 return ((uint32_t)p[0] << 16) | p[1];
99 }
100
101 /**
102 * Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG).
103 * Requires that ce32 is special.
104 */
105 uint32_t getIndirectCE32(uint32_t ce32) const;
106 /**
107 * Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG),
108 * if ce32 is special.
109 */
110 uint32_t getFinalCE32(uint32_t ce32) const;
111
112 /**
113 * Computes a CE from c's ce32 which has the OFFSET_TAG.
114 */
115 int64_t getCEFromOffsetCE32(UChar32 c, uint32_t ce32) const {
116 int64_t dataCE = ces[Collation::indexFromCE32(ce32)];
117 return Collation::makeCE(Collation::getThreeBytePrimaryForOffsetData(c, dataCE));
118 }
119
120 /**
121 * Returns the single CE that c maps to.
122 * Sets U_UNSUPPORTED_ERROR if c does not map to a single CE.
123 */
124 int64_t getSingleCE(UChar32 c, UErrorCode &errorCode) const;
125
126 /**
127 * Returns the FCD16 value for code point c. c must be >= 0.
128 */
129 uint16_t getFCD16(UChar32 c) const {
130 return nfcImpl.getFCD16(c);
131 }
132
133 /**
134 * Returns the first primary for the script's reordering group.
135 * @return the primary with only the first primary lead byte of the group
136 * (not necessarily an actual root collator primary weight),
137 * or 0 if the script is unknown
138 */
139 uint32_t getFirstPrimaryForGroup(int32_t script) const;
140
141 /**
142 * Returns the last primary for the script's reordering group.
143 * @return the last primary of the group
144 * (not an actual root collator primary weight),
145 * or 0 if the script is unknown
146 */
147 uint32_t getLastPrimaryForGroup(int32_t script) const;
148
149 /**
150 * Finds the reordering group which contains the primary weight.
151 * @return the first script of the group, or -1 if the weight is beyond the last group
152 */
153 int32_t getGroupForPrimary(uint32_t p) const;
154
155 int32_t getEquivalentScripts(int32_t script,
156 int32_t dest[], int32_t capacity, UErrorCode &errorCode) const;
157
158 /**
159 * Writes the permutation of primary-weight ranges
160 * for the given reordering of scripts and groups.
161 * The caller checks for illegal arguments and
162 * takes care of [DEFAULT] and memory allocation.
163 *
164 * Each list element will be a (limit, offset) pair as described
165 * for the CollationSettings::reorderRanges.
166 * The list will be empty if no ranges are reordered.
167 */
168 void makeReorderRanges(const int32_t *reorder, int32_t length,
169 UVector32 &ranges, UErrorCode &errorCode) const;
170
171 /** @see jamoCE32s */
172 static const int32_t JAMO_CE32S_LENGTH = 19 + 21 + 27;
173
174 /** Main lookup trie. */
175 const UTrie2 *trie;
176 /**
177 * Array of CE32 values.
178 * At index 0 there must be CE32(U+0000)
179 * to support U+0000's special-tag for NUL-termination handling.
180 */
181 const uint32_t *ce32s;
182 /** Array of CE values for expansions and OFFSET_TAG. */
183 const int64_t *ces;
184 /** Array of prefix and contraction-suffix matching data. */
185 const UChar *contexts;
186 /** Base collation data, or NULL if this data itself is a base. */
187 const CollationData *base;
188 /**
189 * Simple array of JAMO_CE32S_LENGTH=19+21+27 CE32s, one per canonical Jamo L/V/T.
190 * They are normally simple CE32s, rarely expansions.
191 * For fast handling of HANGUL_TAG.
192 */
193 const uint32_t *jamoCE32s;
194 const Normalizer2Impl &nfcImpl;
195 /** The single-byte primary weight (xx000000) for numeric collation. */
196 uint32_t numericPrimary;
197
198 int32_t ce32sLength;
199 int32_t cesLength;
200 int32_t contextsLength;
201
202 /** 256 flags for which primary-weight lead bytes are compressible. */
203 const UBool *compressibleBytes;
204 /**
205 * Set of code points that are unsafe for starting string comparison after an identical prefix,
206 * or in backwards CE iteration.
207 */
208 const UnicodeSet *unsafeBackwardSet;
209
210 /**
211 * Fast Latin table for common-Latin-text string comparisons.
212 * Data structure see class CollationFastLatin.
213 */
214 const uint16_t *fastLatinTable;
215 int32_t fastLatinTableLength;
216
217 /**
218 * Data for scripts and reordering groups.
219 * Uses include building a reordering permutation table and
220 * providing script boundaries to AlphabeticIndex.
221 */
222 int32_t numScripts;
223 /**
224 * The length of scriptsIndex is numScripts+16.
225 * It maps from a UScriptCode or a special reorder code to an entry in scriptStarts.
226 * 16 special reorder codes (not all used) are mapped starting at numScripts.
227 * Up to MAX_NUM_SPECIAL_REORDER_CODES are codes for special groups like space/punct/digit.
228 * There are special codes at the end for reorder-reserved primary ranges.
229 *
230 * Multiple scripts may share a range and index, for example Hira & Kana.
231 */
232 const uint16_t *scriptsIndex;
233 /**
234 * Start primary weight (top 16 bits only) for a group/script/reserved range
235 * indexed by scriptsIndex.
236 * The first range (separators & terminators) and the last range (trailing weights)
237 * are not reorderable, and no scriptsIndex entry points to them.
238 */
239 const uint16_t *scriptStarts;
240 int32_t scriptStartsLength;
241
242 /**
243 * Collation elements in the root collator.
244 * Used by the CollationRootElements class. The data structure is described there.
245 * NULL in a tailoring.
246 */
247 const uint32_t *rootElements;
248 int32_t rootElementsLength;
249
250private:
251 int32_t getScriptIndex(int32_t script) const;
252 void makeReorderRanges(const int32_t *reorder, int32_t length,
253 UBool latinMustMove,
254 UVector32 &ranges, UErrorCode &errorCode) const;
255 int32_t addLowScriptRange(uint8_t table[], int32_t index, int32_t lowStart) const;
256 int32_t addHighScriptRange(uint8_t table[], int32_t index, int32_t highLimit) const;
257};
258
259U_NAMESPACE_END
260
261#endif // !UCONFIG_NO_COLLATION
262#endif // __COLLATIONDATA_H__
263