1// © 2016 and later: Unicode, Inc. and others.
2// License & terms of use: http://www.unicode.org/copyright.html
3/*
4*******************************************************************************
5* Copyright (C) 2013-2015, International Business Machines
6* Corporation and others. All Rights Reserved.
7*******************************************************************************
8* collationdatareader.cpp
9*
10* created on: 2013feb07
11* created by: Markus W. Scherer
12*/
13
14#include "unicode/utypes.h"
15
16#if !UCONFIG_NO_COLLATION
17
18#include "unicode/ucol.h"
19#include "unicode/udata.h"
20#include "unicode/uscript.h"
21#include "cmemory.h"
22#include "collation.h"
23#include "collationdata.h"
24#include "collationdatareader.h"
25#include "collationfastlatin.h"
26#include "collationkeys.h"
27#include "collationrootelements.h"
28#include "collationsettings.h"
29#include "collationtailoring.h"
30#include "collunsafe.h"
31#include "normalizer2impl.h"
32#include "uassert.h"
33#include "ucmndata.h"
34#include "utrie2.h"
35
36U_NAMESPACE_BEGIN
37
38namespace {
39
40int32_t getIndex(const int32_t *indexes, int32_t length, int32_t i) {
41 return (i < length) ? indexes[i] : -1;
42}
43
44} // namespace
45
46void
47CollationDataReader::read(const CollationTailoring *base, const uint8_t *inBytes, int32_t inLength,
48 CollationTailoring &tailoring, UErrorCode &errorCode) {
49 if(U_FAILURE(errorCode)) { return; }
50 if(base != NULL) {
51 if(inBytes == NULL || (0 <= inLength && inLength < 24)) {
52 errorCode = U_ILLEGAL_ARGUMENT_ERROR;
53 return;
54 }
55 const DataHeader *header = reinterpret_cast<const DataHeader *>(inBytes);
56 if(!(header->dataHeader.magic1 == 0xda && header->dataHeader.magic2 == 0x27 &&
57 isAcceptable(tailoring.version, NULL, NULL, &header->info))) {
58 errorCode = U_INVALID_FORMAT_ERROR;
59 return;
60 }
61 if(base->getUCAVersion() != tailoring.getUCAVersion()) {
62 errorCode = U_COLLATOR_VERSION_MISMATCH;
63 return;
64 }
65 int32_t headerLength = header->dataHeader.headerSize;
66 inBytes += headerLength;
67 if(inLength >= 0) {
68 inLength -= headerLength;
69 }
70 }
71
72 if(inBytes == NULL || (0 <= inLength && inLength < 8)) {
73 errorCode = U_ILLEGAL_ARGUMENT_ERROR;
74 return;
75 }
76 const int32_t *inIndexes = reinterpret_cast<const int32_t *>(inBytes);
77 int32_t indexesLength = inIndexes[IX_INDEXES_LENGTH];
78 if(indexesLength < 2 || (0 <= inLength && inLength < indexesLength * 4)) {
79 errorCode = U_INVALID_FORMAT_ERROR; // Not enough indexes.
80 return;
81 }
82
83 // Assume that the tailoring data is in initial state,
84 // with NULL pointers and 0 lengths.
85
86 // Set pointers to non-empty data parts.
87 // Do this in order of their byte offsets. (Should help porting to Java.)
88
89 int32_t index; // one of the indexes[] slots
90 int32_t offset; // byte offset for the index part
91 int32_t length; // number of bytes in the index part
92
93 if(indexesLength > IX_TOTAL_SIZE) {
94 length = inIndexes[IX_TOTAL_SIZE];
95 } else if(indexesLength > IX_REORDER_CODES_OFFSET) {
96 length = inIndexes[indexesLength - 1];
97 } else {
98 length = 0; // only indexes, and inLength was already checked for them
99 }
100 if(0 <= inLength && inLength < length) {
101 errorCode = U_INVALID_FORMAT_ERROR;
102 return;
103 }
104
105 const CollationData *baseData = base == NULL ? NULL : base->data;
106 const int32_t *reorderCodes = NULL;
107 int32_t reorderCodesLength = 0;
108 const uint32_t *reorderRanges = NULL;
109 int32_t reorderRangesLength = 0;
110 index = IX_REORDER_CODES_OFFSET;
111 offset = getIndex(inIndexes, indexesLength, index);
112 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
113 if(length >= 4) {
114 if(baseData == NULL) {
115 // We assume for collation settings that
116 // the base data does not have a reordering.
117 errorCode = U_INVALID_FORMAT_ERROR;
118 return;
119 }
120 reorderCodes = reinterpret_cast<const int32_t *>(inBytes + offset);
121 reorderCodesLength = length / 4;
122
123 // The reorderRanges (if any) are the trailing reorderCodes entries.
124 // Split the array at the boundary.
125 // Script or reorder codes do not exceed 16-bit values.
126 // Range limits are stored in the upper 16 bits, and are never 0.
127 while(reorderRangesLength < reorderCodesLength &&
128 (reorderCodes[reorderCodesLength - reorderRangesLength - 1] & 0xffff0000) != 0) {
129 ++reorderRangesLength;
130 }
131 U_ASSERT(reorderRangesLength < reorderCodesLength);
132 if(reorderRangesLength != 0) {
133 reorderCodesLength -= reorderRangesLength;
134 reorderRanges = reinterpret_cast<const uint32_t *>(reorderCodes + reorderCodesLength);
135 }
136 }
137
138 // There should be a reorder table only if there are reorder codes.
139 // However, when there are reorder codes the reorder table may be omitted to reduce
140 // the data size.
141 const uint8_t *reorderTable = NULL;
142 index = IX_REORDER_TABLE_OFFSET;
143 offset = getIndex(inIndexes, indexesLength, index);
144 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
145 if(length >= 256) {
146 if(reorderCodesLength == 0) {
147 errorCode = U_INVALID_FORMAT_ERROR; // Reordering table without reordering codes.
148 return;
149 }
150 reorderTable = inBytes + offset;
151 } else {
152 // If we have reorder codes, then build the reorderTable at the end,
153 // when the CollationData is otherwise complete.
154 }
155
156 if(baseData != NULL && baseData->numericPrimary != (inIndexes[IX_OPTIONS] & 0xff000000)) {
157 errorCode = U_INVALID_FORMAT_ERROR;
158 return;
159 }
160 CollationData *data = NULL; // Remains NULL if there are no mappings.
161
162 index = IX_TRIE_OFFSET;
163 offset = getIndex(inIndexes, indexesLength, index);
164 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
165 if(length >= 8) {
166 if(!tailoring.ensureOwnedData(errorCode)) { return; }
167 data = tailoring.ownedData;
168 data->base = baseData;
169 data->numericPrimary = inIndexes[IX_OPTIONS] & 0xff000000;
170 data->trie = tailoring.trie = utrie2_openFromSerialized(
171 UTRIE2_32_VALUE_BITS, inBytes + offset, length, NULL,
172 &errorCode);
173 if(U_FAILURE(errorCode)) { return; }
174 } else if(baseData != NULL) {
175 // Use the base data. Only the settings are tailored.
176 tailoring.data = baseData;
177 } else {
178 errorCode = U_INVALID_FORMAT_ERROR; // No mappings.
179 return;
180 }
181
182 index = IX_CES_OFFSET;
183 offset = getIndex(inIndexes, indexesLength, index);
184 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
185 if(length >= 8) {
186 if(data == NULL) {
187 errorCode = U_INVALID_FORMAT_ERROR; // Tailored ces without tailored trie.
188 return;
189 }
190 data->ces = reinterpret_cast<const int64_t *>(inBytes + offset);
191 data->cesLength = length / 8;
192 }
193
194 index = IX_CE32S_OFFSET;
195 offset = getIndex(inIndexes, indexesLength, index);
196 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
197 if(length >= 4) {
198 if(data == NULL) {
199 errorCode = U_INVALID_FORMAT_ERROR; // Tailored ce32s without tailored trie.
200 return;
201 }
202 data->ce32s = reinterpret_cast<const uint32_t *>(inBytes + offset);
203 data->ce32sLength = length / 4;
204 }
205
206 int32_t jamoCE32sStart = getIndex(inIndexes, indexesLength, IX_JAMO_CE32S_START);
207 if(jamoCE32sStart >= 0) {
208 if(data == NULL || data->ce32s == NULL) {
209 errorCode = U_INVALID_FORMAT_ERROR; // Index into non-existent ce32s[].
210 return;
211 }
212 data->jamoCE32s = data->ce32s + jamoCE32sStart;
213 } else if(data == NULL) {
214 // Nothing to do.
215 } else if(baseData != NULL) {
216 data->jamoCE32s = baseData->jamoCE32s;
217 } else {
218 errorCode = U_INVALID_FORMAT_ERROR; // No Jamo CE32s for Hangul processing.
219 return;
220 }
221
222 index = IX_ROOT_ELEMENTS_OFFSET;
223 offset = getIndex(inIndexes, indexesLength, index);
224 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
225 if(length >= 4) {
226 length /= 4;
227 if(data == NULL || length <= CollationRootElements::IX_SEC_TER_BOUNDARIES) {
228 errorCode = U_INVALID_FORMAT_ERROR;
229 return;
230 }
231 data->rootElements = reinterpret_cast<const uint32_t *>(inBytes + offset);
232 data->rootElementsLength = length;
233 uint32_t commonSecTer = data->rootElements[CollationRootElements::IX_COMMON_SEC_AND_TER_CE];
234 if(commonSecTer != Collation::COMMON_SEC_AND_TER_CE) {
235 errorCode = U_INVALID_FORMAT_ERROR;
236 return;
237 }
238 uint32_t secTerBoundaries = data->rootElements[CollationRootElements::IX_SEC_TER_BOUNDARIES];
239 if((secTerBoundaries >> 24) < CollationKeys::SEC_COMMON_HIGH) {
240 // [fixed last secondary common byte] is too low,
241 // and secondary weights would collide with compressed common secondaries.
242 errorCode = U_INVALID_FORMAT_ERROR;
243 return;
244 }
245 }
246
247 index = IX_CONTEXTS_OFFSET;
248 offset = getIndex(inIndexes, indexesLength, index);
249 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
250 if(length >= 2) {
251 if(data == NULL) {
252 errorCode = U_INVALID_FORMAT_ERROR; // Tailored contexts without tailored trie.
253 return;
254 }
255 data->contexts = reinterpret_cast<const UChar *>(inBytes + offset);
256 data->contextsLength = length / 2;
257 }
258
259 index = IX_UNSAFE_BWD_OFFSET;
260 offset = getIndex(inIndexes, indexesLength, index);
261 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
262 if(length >= 2) {
263 if(data == NULL) {
264 errorCode = U_INVALID_FORMAT_ERROR;
265 return;
266 }
267 if(baseData == NULL) {
268#if defined(COLLUNSAFE_COLL_VERSION) && defined (COLLUNSAFE_SERIALIZE)
269 tailoring.unsafeBackwardSet = new UnicodeSet(unsafe_serializedData, unsafe_serializedCount, UnicodeSet::kSerialized, errorCode);
270 if(tailoring.unsafeBackwardSet == NULL) {
271 errorCode = U_MEMORY_ALLOCATION_ERROR;
272 return;
273 } else if (U_FAILURE(errorCode)) {
274 return;
275 }
276#else
277 // Create the unsafe-backward set for the root collator.
278 // Include all non-zero combining marks and trail surrogates.
279 // We do this at load time, rather than at build time,
280 // to simplify Unicode version bootstrapping:
281 // The root data builder only needs the new FractionalUCA.txt data,
282 // but it need not be built with a version of ICU already updated to
283 // the corresponding new Unicode Character Database.
284 //
285 // The following is an optimized version of
286 // new UnicodeSet("[[:^lccc=0:][\\udc00-\\udfff]]").
287 // It is faster and requires fewer code dependencies.
288 tailoring.unsafeBackwardSet = new UnicodeSet(0xdc00, 0xdfff); // trail surrogates
289 if(tailoring.unsafeBackwardSet == NULL) {
290 errorCode = U_MEMORY_ALLOCATION_ERROR;
291 return;
292 }
293 data->nfcImpl.addLcccChars(*tailoring.unsafeBackwardSet);
294#endif // !COLLUNSAFE_SERIALIZE || !COLLUNSAFE_COLL_VERSION
295 } else {
296 // Clone the root collator's set contents.
297 tailoring.unsafeBackwardSet = static_cast<UnicodeSet *>(
298 baseData->unsafeBackwardSet->cloneAsThawed());
299 if(tailoring.unsafeBackwardSet == NULL) {
300 errorCode = U_MEMORY_ALLOCATION_ERROR;
301 return;
302 }
303 }
304 // Add the ranges from the data file to the unsafe-backward set.
305 USerializedSet sset;
306 const uint16_t *unsafeData = reinterpret_cast<const uint16_t *>(inBytes + offset);
307 if(!uset_getSerializedSet(&sset, unsafeData, length / 2)) {
308 errorCode = U_INVALID_FORMAT_ERROR;
309 return;
310 }
311 int32_t count = uset_getSerializedRangeCount(&sset);
312 for(int32_t i = 0; i < count; ++i) {
313 UChar32 start, end;
314 uset_getSerializedRange(&sset, i, &start, &end);
315 tailoring.unsafeBackwardSet->add(start, end);
316 }
317 // Mark each lead surrogate as "unsafe"
318 // if any of its 1024 associated supplementary code points is "unsafe".
319 UChar32 c = 0x10000;
320 for(UChar lead = 0xd800; lead < 0xdc00; ++lead, c += 0x400) {
321 if(!tailoring.unsafeBackwardSet->containsNone(c, c + 0x3ff)) {
322 tailoring.unsafeBackwardSet->add(lead);
323 }
324 }
325 tailoring.unsafeBackwardSet->freeze();
326 data->unsafeBackwardSet = tailoring.unsafeBackwardSet;
327 } else if(data == NULL) {
328 // Nothing to do.
329 } else if(baseData != NULL) {
330 // No tailoring-specific data: Alias the root collator's set.
331 data->unsafeBackwardSet = baseData->unsafeBackwardSet;
332 } else {
333 errorCode = U_INVALID_FORMAT_ERROR; // No unsafeBackwardSet.
334 return;
335 }
336
337 // If the fast Latin format version is different,
338 // or the version is set to 0 for "no fast Latin table",
339 // then just always use the normal string comparison path.
340 if(data != NULL) {
341 data->fastLatinTable = NULL;
342 data->fastLatinTableLength = 0;
343 if(((inIndexes[IX_OPTIONS] >> 16) & 0xff) == CollationFastLatin::VERSION) {
344 index = IX_FAST_LATIN_TABLE_OFFSET;
345 offset = getIndex(inIndexes, indexesLength, index);
346 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
347 if(length >= 2) {
348 data->fastLatinTable = reinterpret_cast<const uint16_t *>(inBytes + offset);
349 data->fastLatinTableLength = length / 2;
350 if((*data->fastLatinTable >> 8) != CollationFastLatin::VERSION) {
351 errorCode = U_INVALID_FORMAT_ERROR; // header vs. table version mismatch
352 return;
353 }
354 } else if(baseData != NULL) {
355 data->fastLatinTable = baseData->fastLatinTable;
356 data->fastLatinTableLength = baseData->fastLatinTableLength;
357 }
358 }
359 }
360
361 index = IX_SCRIPTS_OFFSET;
362 offset = getIndex(inIndexes, indexesLength, index);
363 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
364 if(length >= 2) {
365 if(data == NULL) {
366 errorCode = U_INVALID_FORMAT_ERROR;
367 return;
368 }
369 const uint16_t *scripts = reinterpret_cast<const uint16_t *>(inBytes + offset);
370 int32_t scriptsLength = length / 2;
371 data->numScripts = scripts[0];
372 // There must be enough entries for both arrays, including more than two range starts.
373 data->scriptStartsLength = scriptsLength - (1 + data->numScripts + 16);
374 if(data->scriptStartsLength <= 2 ||
375 CollationData::MAX_NUM_SCRIPT_RANGES < data->scriptStartsLength) {
376 errorCode = U_INVALID_FORMAT_ERROR;
377 return;
378 }
379 data->scriptsIndex = scripts + 1;
380 data->scriptStarts = scripts + 1 + data->numScripts + 16;
381 if(!(data->scriptStarts[0] == 0 &&
382 data->scriptStarts[1] == ((Collation::MERGE_SEPARATOR_BYTE + 1) << 8) &&
383 data->scriptStarts[data->scriptStartsLength - 1] ==
384 (Collation::TRAIL_WEIGHT_BYTE << 8))) {
385 errorCode = U_INVALID_FORMAT_ERROR;
386 return;
387 }
388 } else if(data == NULL) {
389 // Nothing to do.
390 } else if(baseData != NULL) {
391 data->numScripts = baseData->numScripts;
392 data->scriptsIndex = baseData->scriptsIndex;
393 data->scriptStarts = baseData->scriptStarts;
394 data->scriptStartsLength = baseData->scriptStartsLength;
395 }
396
397 index = IX_COMPRESSIBLE_BYTES_OFFSET;
398 offset = getIndex(inIndexes, indexesLength, index);
399 length = getIndex(inIndexes, indexesLength, index + 1) - offset;
400 if(length >= 256) {
401 if(data == NULL) {
402 errorCode = U_INVALID_FORMAT_ERROR;
403 return;
404 }
405 data->compressibleBytes = reinterpret_cast<const UBool *>(inBytes + offset);
406 } else if(data == NULL) {
407 // Nothing to do.
408 } else if(baseData != NULL) {
409 data->compressibleBytes = baseData->compressibleBytes;
410 } else {
411 errorCode = U_INVALID_FORMAT_ERROR; // No compressibleBytes[].
412 return;
413 }
414
415 const CollationSettings &ts = *tailoring.settings;
416 int32_t options = inIndexes[IX_OPTIONS] & 0xffff;
417 uint16_t fastLatinPrimaries[CollationFastLatin::LATIN_LIMIT];
418 int32_t fastLatinOptions = CollationFastLatin::getOptions(
419 tailoring.data, ts, fastLatinPrimaries, UPRV_LENGTHOF(fastLatinPrimaries));
420 if(options == ts.options && ts.variableTop != 0 &&
421 reorderCodesLength == ts.reorderCodesLength &&
422 (reorderCodesLength == 0 ||
423 uprv_memcmp(reorderCodes, ts.reorderCodes, reorderCodesLength * 4) == 0) &&
424 fastLatinOptions == ts.fastLatinOptions &&
425 (fastLatinOptions < 0 ||
426 uprv_memcmp(fastLatinPrimaries, ts.fastLatinPrimaries,
427 sizeof(fastLatinPrimaries)) == 0)) {
428 return;
429 }
430
431 CollationSettings *settings = SharedObject::copyOnWrite(tailoring.settings);
432 if(settings == NULL) {
433 errorCode = U_MEMORY_ALLOCATION_ERROR;
434 return;
435 }
436 settings->options = options;
437 // Set variableTop from options and scripts data.
438 settings->variableTop = tailoring.data->getLastPrimaryForGroup(
439 UCOL_REORDER_CODE_FIRST + settings->getMaxVariable());
440 if(settings->variableTop == 0) {
441 errorCode = U_INVALID_FORMAT_ERROR;
442 return;
443 }
444
445 if(reorderCodesLength != 0) {
446 settings->aliasReordering(*baseData, reorderCodes, reorderCodesLength,
447 reorderRanges, reorderRangesLength,
448 reorderTable, errorCode);
449 }
450
451 settings->fastLatinOptions = CollationFastLatin::getOptions(
452 tailoring.data, *settings,
453 settings->fastLatinPrimaries, UPRV_LENGTHOF(settings->fastLatinPrimaries));
454}
455
456UBool U_CALLCONV
457CollationDataReader::isAcceptable(void *context,
458 const char * /* type */, const char * /*name*/,
459 const UDataInfo *pInfo) {
460 if(
461 pInfo->size >= 20 &&
462 pInfo->isBigEndian == U_IS_BIG_ENDIAN &&
463 pInfo->charsetFamily == U_CHARSET_FAMILY &&
464 pInfo->dataFormat[0] == 0x55 && // dataFormat="UCol"
465 pInfo->dataFormat[1] == 0x43 &&
466 pInfo->dataFormat[2] == 0x6f &&
467 pInfo->dataFormat[3] == 0x6c &&
468 pInfo->formatVersion[0] == 5
469 ) {
470 UVersionInfo *version = static_cast<UVersionInfo *>(context);
471 if(version != NULL) {
472 uprv_memcpy(version, pInfo->dataVersion, 4);
473 }
474 return TRUE;
475 } else {
476 return FALSE;
477 }
478}
479
480U_NAMESPACE_END
481
482#endif // !UCONFIG_NO_COLLATION
483