1//************************************ bs::framework - Copyright 2018 Marko Pintera **************************************//
2//*********** Licensed under the MIT license. See LICENSE.md for full terms. This notice is not to be removed. ***********//
3#include "Serialization/BsBinarySerializer.h"
4
5#include "Error/BsException.h"
6#include "Debug/BsDebug.h"
7#include "Reflection/BsIReflectable.h"
8#include "Reflection/BsRTTIType.h"
9#include "Reflection/BsRTTIField.h"
10#include "Reflection/BsRTTIPlainField.h"
11#include "Reflection/BsRTTIReflectableField.h"
12#include "Reflection/BsRTTIReflectablePtrField.h"
13#include "Reflection/BsRTTIManagedDataBlockField.h"
14#include "Serialization/BsMemorySerializer.h"
15#include "FileSystem/BsDataStream.h"
16
17namespace bs
18{
19
20 /**
21 * A macro that represents a block of code that gets used a lot inside encodeInternal. It checks if the buffer has enough
22 * space, and if it does it copies the data from the specified location and increments the needed pointers and counters. If
23 * there is not enough space the buffer is flushed (hopefully to make some space). If there is still not enough space the
24 * entire encoding process ends.
25 *
26 * @param dataPtr Pointer to data which to copy.
27 * @param size Size of the data to copy
28 */
29#define COPY_TO_BUFFER(dataIter, size) \
30if((*bytesWritten + size) > bufferLength) \
31{ \
32 mTotalBytesWritten += *bytesWritten; \
33 buffer = flushBufferCallback(buffer - *bytesWritten, *bytesWritten, bufferLength); \
34 if(buffer == nullptr || bufferLength < size) return nullptr; \
35 *bytesWritten = 0; \
36} \
37 \
38memcpy(buffer, dataIter, size); \
39buffer += size; \
40*bytesWritten += size;
41
42/** Reports that @p size bytes were read from the data buffer */
43#define REPORT_READ(size) \
44{ \
45 mTotalBytesRead += size; \
46 if(mReportProgress && (mTotalBytesRead >= mNextProgressReport)) \
47 { \
48 UINT32 lastReport = (mTotalBytesRead / REPORT_AFTER_BYTES) * REPORT_AFTER_BYTES; \
49 mNextProgressReport = lastReport + REPORT_AFTER_BYTES; \
50 \
51 mReportProgress(mTotalBytesRead / (float)mTotalBytesToRead); \
52 } \
53}
54/** Reads from the data buffer into the provided output and advances the read position. */
55#define READ_FROM_BUFFER(output, size) \
56{ \
57 if(data->read(output, size) != size) \
58 { \
59 BS_EXCEPT(InternalErrorException, "Error decoding data."); \
60 } \
61 \
62 REPORT_READ(size) \
63}
64
65/** Skips the next @p size bytes data buffer and advances the read position. */
66#define SKIP_READ(size) \
67{ \
68 data->skip(size); \
69 REPORT_READ(size) \
70}
71
72/** Moves the current read buffer read position back @p size bytes. */
73#define SEEK_BACK(size) \
74data->seek(data->tell() - size); \
75mTotalBytesRead -= size; \
76
77 constexpr UINT32 BinarySerializer::REPORT_AFTER_BYTES;
78
79 BinarySerializer::BinarySerializer()
80 :mAlloc(&gFrameAlloc())
81 { }
82
83 void BinarySerializer::encode(IReflectable* object, UINT8* buffer, UINT32 bufferLength, UINT32* bytesWritten,
84 std::function<UINT8*(UINT8*, UINT32, UINT32&)> flushBufferCallback, bool shallow, SerializationContext* context)
85 {
86 mObjectsToEncode.clear();
87 mObjectAddrToId.clear();
88 mLastUsedObjectId = 1;
89 *bytesWritten = 0;
90 mTotalBytesWritten = 0;
91 mContext = context;
92
93 mAlloc->markFrame();
94
95 Vector<SPtr<IReflectable>> encodedObjects;
96 UINT32 objectId = findOrCreatePersistentId(object);
97
98 // Encode primary object and its value types
99 buffer = encodeEntry(object, objectId, buffer, bufferLength, bytesWritten, flushBufferCallback, shallow);
100 if(buffer == nullptr)
101 {
102 BS_EXCEPT(InternalErrorException,
103 "Destination buffer is null or not large enough.");
104 }
105
106 // Encode pointed to objects and their value types
107 UnorderedSet<UINT32> serializedObjects;
108 while(true)
109 {
110 auto iter = mObjectsToEncode.begin();
111 bool foundObjectToProcess = false;
112 for(; iter != mObjectsToEncode.end(); ++iter)
113 {
114 auto foundExisting = serializedObjects.find(iter->objectId);
115 if(foundExisting != serializedObjects.end())
116 continue; // Already processed
117
118 SPtr<IReflectable> curObject = iter->object;
119 UINT32 curObjectid = iter->objectId;
120 serializedObjects.insert(curObjectid);
121 mObjectsToEncode.erase(iter);
122
123 buffer = encodeEntry(curObject.get(), curObjectid, buffer,
124 bufferLength, bytesWritten, flushBufferCallback, shallow);
125 if(buffer == nullptr)
126 {
127 BS_EXCEPT(InternalErrorException,
128 "Destination buffer is null or not large enough.");
129 }
130
131 foundObjectToProcess = true;
132
133 // Ensure we keep a reference to the object so it isn't released.
134 // The system assigns unique IDs to IReflectable objects based on pointer
135 // addresses but if objects get released then same address could be assigned twice.
136 // Note: To get around this I could assign unique IDs to IReflectable objects
137 encodedObjects.push_back(curObject);
138
139 break; // Need to start over as mObjectsToSerialize was possibly modified
140 }
141
142 if(!foundObjectToProcess) // We're done
143 break;
144 }
145
146 // Final flush
147 if(*bytesWritten > 0)
148 {
149 mTotalBytesWritten += *bytesWritten;
150 buffer = flushBufferCallback(buffer - *bytesWritten, *bytesWritten, bufferLength);
151 }
152
153 *bytesWritten = mTotalBytesWritten;
154
155 encodedObjects.clear();
156 mObjectsToEncode.clear();
157 mObjectAddrToId.clear();
158
159 mAlloc->clear();
160 }
161
162 SPtr<IReflectable> BinarySerializer::decode(const SPtr<DataStream>& data, UINT32 dataLength,
163 SerializationContext* context, std::function<void(float)> progress)
164 {
165 mContext = context;
166 mReportProgress = nullptr;
167 mTotalBytesToRead = dataLength;
168 mTotalBytesRead = 0;
169
170 if (dataLength == 0)
171 {
172 if(mReportProgress)
173 mReportProgress(1.0f);
174
175 return nullptr;
176 }
177
178 const size_t start = data->tell();
179 const size_t end = start + dataLength;
180 mDecodeObjectMap.clear();
181
182 // Note: Ideally we can avoid iterating twice over the stream data
183 // Create empty instances of all ptr objects
184 SPtr<IReflectable> rootObject = nullptr;
185 do
186 {
187 ObjectMetaData objectMetaData;
188 objectMetaData.objectMeta = 0;
189 objectMetaData.typeId = 0;
190
191 if(data->read(&objectMetaData, sizeof(ObjectMetaData)) != sizeof(ObjectMetaData))
192 {
193 BS_EXCEPT(InternalErrorException, "Error decoding data.");
194 }
195
196 data->seek(data->tell() - sizeof(ObjectMetaData));
197
198 UINT32 objectId = 0;
199 UINT32 objectTypeId = 0;
200 bool objectIsBaseClass = false;
201 decodeObjectMetaData(objectMetaData, objectId, objectTypeId, objectIsBaseClass);
202
203 if (objectIsBaseClass)
204 {
205 BS_EXCEPT(InternalErrorException, "Encountered a base-class object while looking for a new object. " \
206 "Base class objects are only supposed to be parts of a larger object.");
207 }
208
209 SPtr<IReflectable> object = IReflectable::createInstanceFromTypeId(objectTypeId);
210 mDecodeObjectMap.insert(std::make_pair(objectId, ObjectToDecode(object, data->tell())));
211
212 if(rootObject == nullptr)
213 rootObject = object;
214
215 } while (decodeEntry(data, end, nullptr));
216
217 assert(mTotalBytesRead == mTotalBytesToRead);
218
219 // Don't set report callback until we actually do the reads
220 mReportProgress = std::move(progress);
221 mTotalBytesRead = 0;
222
223 // Now go through all of the objects and actually decode them
224 for(auto iter = mDecodeObjectMap.begin(); iter != mDecodeObjectMap.end(); ++iter)
225 {
226 ObjectToDecode& objToDecode = iter->second;
227
228 if(objToDecode.isDecoded)
229 continue;
230
231 data->seek(objToDecode.offset);
232
233 objToDecode.decodeInProgress = true;
234 decodeEntry(data, end, objToDecode.object);
235 objToDecode.decodeInProgress = false;
236 objToDecode.isDecoded = true;
237 }
238
239 mDecodeObjectMap.clear();
240 data->seek(end);
241
242 assert(mTotalBytesRead == mTotalBytesToRead);
243
244 if(mReportProgress)
245 mReportProgress(1.0f);
246
247 return rootObject;
248 }
249
250 UINT8* BinarySerializer::encodeEntry(IReflectable* object, UINT32 objectId, UINT8* buffer, UINT32& bufferLength,
251 UINT32* bytesWritten, std::function<UINT8*(UINT8*, UINT32, UINT32&)> flushBufferCallback, bool shallow)
252 {
253 RTTITypeBase* rtti = object->getRTTI();
254 bool isBaseClass = false;
255
256 FrameStack<RTTITypeBase*> rttiInstances;
257
258 const auto cleanup = [&]()
259 {
260 while (!rttiInstances.empty())
261 {
262 RTTITypeBase* rttiInstance = rttiInstances.top();
263 rttiInstance->onSerializationEnded(object, mContext);
264 mAlloc->destruct(rttiInstance);
265
266 rttiInstances.pop();
267 }
268 };
269
270 // If an object has base classes, we need to iterate through all of them
271 do
272 {
273 RTTITypeBase* rttiInstance = rtti->_clone(*mAlloc);
274 rttiInstances.push(rttiInstance);
275
276 rttiInstance->onSerializationStarted(object, mContext);
277
278 // Encode object ID & type
279 ObjectMetaData objectMetaData = encodeObjectMetaData(objectId, rtti->getRTTIId(), isBaseClass);
280 COPY_TO_BUFFER(&objectMetaData, sizeof(ObjectMetaData))
281
282 const UINT32 numFields = rtti->getNumFields();
283 for(UINT32 i = 0; i < numFields; i++)
284 {
285 RTTIField* curGenericField = rtti->getField(i);
286
287 // Copy field ID & other meta-data like field size and type
288 int metaData = encodeFieldMetaData(curGenericField->mUniqueId, curGenericField->getTypeSize(),
289 curGenericField->mIsVectorType, curGenericField->mType, curGenericField->hasDynamicSize(), false);
290 COPY_TO_BUFFER(&metaData, META_SIZE)
291
292 if(curGenericField->mIsVectorType)
293 {
294 UINT32 arrayNumElems = curGenericField->getArraySize(rttiInstance, object);
295
296 // Copy num vector elements
297 COPY_TO_BUFFER(&arrayNumElems, NUM_ELEM_FIELD_SIZE)
298
299 switch(curGenericField->mType)
300 {
301 case SerializableFT_ReflectablePtr:
302 {
303 RTTIReflectablePtrFieldBase* curField = static_cast<RTTIReflectablePtrFieldBase*>(curGenericField);
304
305 for(UINT32 arrIdx = 0; arrIdx < arrayNumElems; arrIdx++)
306 {
307 SPtr<IReflectable> childObject;
308
309 if (!shallow)
310 childObject = curField->getArrayValue(rttiInstance, object, arrIdx);
311
312 UINT32 objId = registerObjectPtr(childObject);
313 COPY_TO_BUFFER(&objId, sizeof(UINT32))
314 }
315
316 break;
317 }
318 case SerializableFT_Reflectable:
319 {
320 RTTIReflectableFieldBase* curField = static_cast<RTTIReflectableFieldBase*>(curGenericField);
321
322 for(UINT32 arrIdx = 0; arrIdx < arrayNumElems; arrIdx++)
323 {
324 IReflectable& childObject = curField->getArrayValue(rttiInstance, object, arrIdx);
325
326 buffer = complexTypeToBuffer(&childObject, buffer, bufferLength,
327 bytesWritten, flushBufferCallback, shallow);
328 if(buffer == nullptr)
329 {
330 cleanup();
331 return nullptr;
332 }
333 }
334
335 break;
336 }
337 case SerializableFT_Plain:
338 {
339 RTTIPlainFieldBase* curField = static_cast<RTTIPlainFieldBase*>(curGenericField);
340
341 for(UINT32 arrIdx = 0; arrIdx < arrayNumElems; arrIdx++)
342 {
343 UINT32 typeSize = 0;
344 if(curField->hasDynamicSize())
345 typeSize = curField->getArrayElemDynamicSize(rttiInstance, object, arrIdx);
346 else
347 typeSize = curField->getTypeSize();
348
349 if ((*bytesWritten + typeSize) > bufferLength)
350 {
351 UINT8* tempBuffer = (UINT8*)bs_stack_alloc(typeSize);
352 curField->arrayElemToBuffer(rttiInstance, object, arrIdx, tempBuffer);
353
354 buffer = dataBlockToBuffer(tempBuffer, typeSize, buffer, bufferLength, bytesWritten, flushBufferCallback);
355 bs_stack_free(tempBuffer);
356
357 if (buffer == nullptr || bufferLength == 0)
358 {
359 cleanup();
360 return nullptr;
361 }
362 }
363 else
364 {
365 curField->arrayElemToBuffer(rttiInstance, object, arrIdx, buffer);
366 buffer += typeSize;
367 *bytesWritten += typeSize;
368 }
369 }
370
371 break;
372 }
373 default:
374 BS_EXCEPT(InternalErrorException,
375 "Error encoding data. Encountered a type I don't know how to encode. Type: " + toString(UINT32(curGenericField->mType)) +
376 ", Is array: " + toString(curGenericField->mIsVectorType));
377 }
378 }
379 else
380 {
381 switch(curGenericField->mType)
382 {
383 case SerializableFT_ReflectablePtr:
384 {
385 RTTIReflectablePtrFieldBase* curField = static_cast<RTTIReflectablePtrFieldBase*>(curGenericField);
386 SPtr<IReflectable> childObject;
387
388 if (!shallow)
389 childObject = curField->getValue(rttiInstance, object);
390
391 UINT32 objId = registerObjectPtr(childObject);
392 COPY_TO_BUFFER(&objId, sizeof(UINT32))
393
394 break;
395 }
396 case SerializableFT_Reflectable:
397 {
398 RTTIReflectableFieldBase* curField = static_cast<RTTIReflectableFieldBase*>(curGenericField);
399 IReflectable& childObject = curField->getValue(rttiInstance, object);
400
401 buffer = complexTypeToBuffer(&childObject, buffer, bufferLength,
402 bytesWritten, flushBufferCallback, shallow);
403 if(buffer == nullptr)
404 {
405 cleanup();
406 return nullptr;
407 }
408
409 break;
410 }
411 case SerializableFT_Plain:
412 {
413 RTTIPlainFieldBase* curField = static_cast<RTTIPlainFieldBase*>(curGenericField);
414
415 UINT32 typeSize = 0;
416 if(curField->hasDynamicSize())
417 typeSize = curField->getDynamicSize(rttiInstance, object);
418 else
419 typeSize = curField->getTypeSize();
420
421 if ((*bytesWritten + typeSize) > bufferLength)
422 {
423 UINT8* tempBuffer = (UINT8*)bs_stack_alloc(typeSize);
424 curField->toBuffer(rttiInstance, object, tempBuffer);
425
426 buffer = dataBlockToBuffer(tempBuffer, typeSize, buffer, bufferLength, bytesWritten, flushBufferCallback);
427 bs_stack_free(tempBuffer);
428
429 if (buffer == nullptr || bufferLength == 0)
430 {
431 cleanup();
432 return nullptr;
433 }
434 }
435 else
436 {
437 curField->toBuffer(rttiInstance, object, buffer);
438 buffer += typeSize;
439 *bytesWritten += typeSize;
440 }
441
442 break;
443 }
444 case SerializableFT_DataBlock:
445 {
446 RTTIManagedDataBlockFieldBase* curField = static_cast<RTTIManagedDataBlockFieldBase*>(curGenericField);
447
448 UINT32 dataBlockSize = 0;
449 SPtr<DataStream> blockStream = curField->getValue(rttiInstance, object, dataBlockSize);
450
451 // Data block size
452 COPY_TO_BUFFER(&dataBlockSize, sizeof(UINT32))
453
454 // Data block data
455 UINT8* dataToStore = (UINT8*)bs_stack_alloc(dataBlockSize);
456 blockStream->read(dataToStore, dataBlockSize);
457
458 buffer = dataBlockToBuffer(dataToStore, dataBlockSize, buffer, bufferLength, bytesWritten, flushBufferCallback);
459 bs_stack_free(dataToStore);
460
461 if (buffer == nullptr || bufferLength == 0)
462 {
463 cleanup();
464 return nullptr;
465 }
466
467 break;
468 }
469 default:
470 BS_EXCEPT(InternalErrorException,
471 "Error encoding data. Encountered a type I don't know how to encode. Type: " + toString(UINT32(curGenericField->mType)) +
472 ", Is array: " + toString(curGenericField->mIsVectorType));
473 }
474 }
475 }
476
477 rtti = rtti->getBaseClass();
478 isBaseClass = true;
479
480 } while(rtti != nullptr); // Repeat until we reach the top of the inheritance hierarchy
481
482 cleanup();
483
484 return buffer;
485 }
486
487 bool BinarySerializer::decodeEntry(const SPtr<DataStream>& data, size_t dataEnd, const SPtr<IReflectable>& output)
488 {
489 ObjectMetaData objectMetaData;
490 objectMetaData.objectMeta = 0;
491 objectMetaData.typeId = 0;
492
493 READ_FROM_BUFFER(&objectMetaData, sizeof(ObjectMetaData))
494
495 UINT32 objectId = 0;
496 UINT32 objectTypeId = 0;
497 bool objectIsBaseClass = false;
498 decodeObjectMetaData(objectMetaData, objectId, objectTypeId, objectIsBaseClass);
499
500 if (objectIsBaseClass)
501 {
502 BS_EXCEPT(InternalErrorException, "Encountered a base-class object while looking for a new object. " \
503 "Base class objects are only supposed to be parts of a larger object.");
504 }
505
506 RTTITypeBase* rtti = nullptr;
507 if(output)
508 rtti = output->getRTTI();
509
510 FrameVector<RTTITypeBase*> rttiInstances;
511
512 auto finalizeObject = [&rttiInstances, this](IReflectable* object)
513 {
514 // Note: It would make sense to finish deserializing derived classes before base classes, but some code
515 // depends on the old functionality, so we'll keep it this way
516 for (auto iter = rttiInstances.rbegin(); iter != rttiInstances.rend(); ++iter)
517 {
518 RTTITypeBase* curRTTI = *iter;
519
520 curRTTI->onDeserializationEnded(object, mContext);
521 mAlloc->destruct(curRTTI);
522 }
523
524 rttiInstances.clear();
525 };
526
527 RTTITypeBase* curRTTI = rtti;
528 while (curRTTI)
529 {
530 RTTITypeBase* rttiInstance = curRTTI->_clone(*mAlloc);
531 rttiInstances.push_back(rttiInstance);
532
533 curRTTI = curRTTI->getBaseClass();
534 }
535
536 // Iterate in reverse to notify base classes before derived classes
537 for(auto iter = rttiInstances.rbegin(); iter != rttiInstances.rend(); ++iter)
538 (*iter)->onDeserializationStarted(output.get(), mContext);
539
540 RTTITypeBase* rttiInstance = nullptr;
541 UINT32 rttiInstanceIdx = 0;
542 if(!rttiInstances.empty())
543 rttiInstance = rttiInstances[0];
544
545 while (data->tell() < dataEnd)
546 {
547 int metaData = -1;
548 READ_FROM_BUFFER(&metaData, META_SIZE)
549
550 if (isObjectMetaData(metaData)) // We've reached a new object or a base class of the current one
551 {
552 ObjectMetaData objMetaData;
553 objMetaData.objectMeta = 0;
554 objMetaData.typeId = 0;
555
556 SEEK_BACK(META_SIZE)
557 READ_FROM_BUFFER(&objMetaData, sizeof(ObjectMetaData))
558
559 UINT32 objId = 0;
560 UINT32 objTypeId = 0;
561 bool objIsBaseClass = false;
562 decodeObjectMetaData(objMetaData, objId, objTypeId, objIsBaseClass);
563
564 // If it's a base class, get base class RTTI and handle that
565 if (objIsBaseClass)
566 {
567 if (rtti != nullptr)
568 rtti = rtti->getBaseClass();
569
570 // Saved and current base classes don't match, so just skip over all that data
571 if (rtti == nullptr || rtti->getRTTIId() != objTypeId)
572 rtti = nullptr;
573
574 rttiInstance = nullptr;
575
576 if (rtti)
577 {
578 rttiInstance = rttiInstances[rttiInstanceIdx + 1];
579 rttiInstanceIdx++;
580 }
581
582 continue;
583 }
584 else
585 {
586 // Found new object, we're done
587 SEEK_BACK(sizeof(ObjectMetaData))
588
589 finalizeObject(output.get());
590 return true;
591 }
592 }
593
594 bool isArray;
595 SerializableFieldType fieldType;
596 UINT16 fieldId;
597 UINT8 fieldSize;
598 bool hasDynamicSize;
599 bool terminator;
600 decodeFieldMetaData(metaData, fieldId, fieldSize, isArray, fieldType, hasDynamicSize, terminator);
601
602 if (terminator)
603 {
604 // We've processed the last field in this object, so return. Although we return false we don't actually know
605 // if there is an object following this one. However it doesn't matter since terminator fields are only used
606 // for embedded objects that are all processed within this method so we can compensate.
607 finalizeObject(output.get());
608 return false;
609 }
610
611 RTTIField* curGenericField = nullptr;
612
613 if (rtti != nullptr)
614 curGenericField = rtti->findField(fieldId);
615
616 if (curGenericField != nullptr)
617 {
618 if (!hasDynamicSize && curGenericField->getTypeSize() != fieldSize)
619 {
620 BS_EXCEPT(InternalErrorException,
621 "Data type mismatch. Type size stored in file and actual type size don't match. ("
622 + toString(curGenericField->getTypeSize()) + " vs. " + toString(fieldSize) + ")");
623 }
624
625 if (curGenericField->mIsVectorType != isArray)
626 {
627 BS_EXCEPT(InternalErrorException,
628 "Data type mismatch. One is array, other is a single type.");
629 }
630
631 if (curGenericField->mType != fieldType)
632 {
633 BS_EXCEPT(InternalErrorException,
634 "Data type mismatch. Field types don't match. " + toString(UINT32(curGenericField->mType)) + " vs. " + toString(UINT32(fieldType)));
635 }
636 }
637
638 int arrayNumElems = 1;
639 if (isArray)
640 {
641 READ_FROM_BUFFER(&arrayNumElems, NUM_ELEM_FIELD_SIZE)
642
643 if(curGenericField != nullptr)
644 curGenericField->setArraySize(rttiInstance, output.get(), arrayNumElems);
645
646 switch (fieldType)
647 {
648 case SerializableFT_ReflectablePtr:
649 {
650 RTTIReflectablePtrFieldBase* curField = static_cast<RTTIReflectablePtrFieldBase*>(curGenericField);
651
652 for (int i = 0; i < arrayNumElems; i++)
653 {
654 int childObjectId = 0;
655 READ_FROM_BUFFER(&childObjectId, COMPLEX_TYPE_FIELD_SIZE)
656
657 if (curField != nullptr)
658 {
659 auto findObj = mDecodeObjectMap.find(childObjectId);
660
661 if(findObj == mDecodeObjectMap.end())
662 {
663 if(childObjectId != 0)
664 {
665 LOGWRN("When deserializing, object ID: " + toString(childObjectId) +
666 " was found but no such object was contained in the file.");
667 }
668
669 curField->setArrayValue(rttiInstance, output.get(), i, nullptr);
670 }
671 else
672 {
673 ObjectToDecode& objToDecode = findObj->second;
674
675 const bool needsDecoding = !curField->getInfo().flags.isSet(RTTIFieldFlag::WeakRef) && !objToDecode.isDecoded;
676 if (needsDecoding)
677 {
678 if (objToDecode.decodeInProgress)
679 {
680 LOGWRN("Detected a circular reference when decoding. Referenced object's fields " \
681 "will be resolved in an undefined order (i.e. one of the objects will not " \
682 "be fully deserialized when assigned to its field). Use RTTI_Flag_WeakRef to " \
683 "get rid of this warning and tell the system which of the objects is allowed " \
684 "to be deserialized after it is assigned to its field.");
685 }
686 else
687 {
688 objToDecode.decodeInProgress = true;
689
690 const size_t curOffset = data->tell();
691 data->seek(objToDecode.offset);
692 decodeEntry(data, dataEnd, objToDecode.object);
693 data->seek(curOffset);
694
695 objToDecode.decodeInProgress = false;
696 objToDecode.isDecoded = true;
697 }
698 }
699
700 curField->setArrayValue(rttiInstance, output.get(), i, objToDecode.object);
701 }
702 }
703 }
704
705 break;
706 }
707 case SerializableFT_Reflectable:
708 {
709 RTTIReflectableFieldBase* curField = static_cast<RTTIReflectableFieldBase*>(curGenericField);
710
711 for (int i = 0; i < arrayNumElems; i++)
712 {
713 SPtr<IReflectable> childObj;
714 if(curField)
715 childObj = curField->newObject();
716
717 decodeEntry(data, dataEnd, childObj);
718
719 if (curField != nullptr)
720 {
721 // Note: Would be nice to avoid this copy by value and decode directly into the field
722 curField->setArrayValue(rttiInstance, output.get(), i, *childObj);
723 }
724 }
725 break;
726 }
727 case SerializableFT_Plain:
728 {
729 RTTIPlainFieldBase* curField = static_cast<RTTIPlainFieldBase*>(curGenericField);
730
731 for (int i = 0; i < arrayNumElems; i++)
732 {
733 UINT32 typeSize = fieldSize;
734 if (hasDynamicSize)
735 {
736 data->read(&typeSize, sizeof(UINT32));
737 data->seek(data->tell() - sizeof(UINT32));
738 }
739
740 if (curField != nullptr)
741 {
742 // Note: Two data copies that can potentially be avoided:
743 // - Copy from stream into a temporary buffer (use stream directly for decoding)
744 // - Internally the field will do a value copy of the decoded object (ideally we decode directly into the destination)
745 void* fieldValue = bs_stack_alloc(typeSize);
746 READ_FROM_BUFFER(fieldValue, typeSize)
747
748 curField->arrayElemFromBuffer(rttiInstance, output.get(), i, fieldValue);
749 bs_stack_free(fieldValue);
750 }
751 else
752 SKIP_READ(typeSize);
753 }
754 break;
755 }
756 default:
757 BS_EXCEPT(InternalErrorException,
758 "Error decoding data. Encountered a type I don't know how to decode. Type: " + toString(UINT32(fieldType)) +
759 ", Is array: " + toString(isArray));
760 }
761 }
762 else
763 {
764 switch (fieldType)
765 {
766 case SerializableFT_ReflectablePtr:
767 {
768 RTTIReflectablePtrFieldBase* curField = static_cast<RTTIReflectablePtrFieldBase*>(curGenericField);
769
770 int childObjectId = 0;
771 READ_FROM_BUFFER(&childObjectId, COMPLEX_TYPE_FIELD_SIZE)
772
773 if (curField != nullptr)
774 {
775 auto findObj = mDecodeObjectMap.find(childObjectId);
776
777 if(findObj == mDecodeObjectMap.end())
778 {
779 if(childObjectId != 0)
780 {
781 LOGWRN("When deserializing, object ID: " + toString(childObjectId) +
782 " was found but no such object was contained in the file.");
783 }
784
785 curField->setValue(rttiInstance, output.get(), nullptr);
786 }
787 else
788 {
789 ObjectToDecode& objToDecode = findObj->second;
790
791 const bool needsDecoding = !curField->getInfo().flags.isSet(RTTIFieldFlag::WeakRef) && !objToDecode.isDecoded;
792 if (needsDecoding)
793 {
794 if (objToDecode.decodeInProgress)
795 {
796 LOGWRN("Detected a circular reference when decoding. Referenced object's fields " \
797 "will be resolved in an undefined order (i.e. one of the objects will not " \
798 "be fully deserialized when assigned to its field). Use RTTI_Flag_WeakRef to " \
799 "get rid of this warning and tell the system which of the objects is allowed " \
800 "to be deserialized after it is assigned to its field.");
801 }
802 else
803 {
804 objToDecode.decodeInProgress = true;
805
806 const size_t curOffset = data->tell();
807 data->seek(objToDecode.offset);
808 decodeEntry(data, dataEnd, objToDecode.object);
809 data->seek(curOffset);
810
811 objToDecode.decodeInProgress = false;
812 objToDecode.isDecoded = true;
813 }
814 }
815
816 curField->setValue(rttiInstance, output.get(), objToDecode.object);
817 }
818 }
819
820 break;
821 }
822 case SerializableFT_Reflectable:
823 {
824 RTTIReflectableFieldBase* curField = static_cast<RTTIReflectableFieldBase*>(curGenericField);
825
826 // Note: Ideally we can skip decoding the entry if the field no longer exists
827 SPtr<IReflectable> childObj;
828 if (curField)
829 childObj = curField->newObject();
830
831 decodeEntry(data, dataEnd, childObj);
832
833 if (curField != nullptr)
834 {
835 // Note: Would be nice to avoid this copy by value and decode directly into the field
836 curField->setValue(rttiInstance, output.get(), *childObj);
837 }
838
839 break;
840 }
841 case SerializableFT_Plain:
842 {
843 RTTIPlainFieldBase* curField = static_cast<RTTIPlainFieldBase*>(curGenericField);
844
845 UINT32 typeSize = fieldSize;
846 if (hasDynamicSize)
847 {
848 data->read(&typeSize, sizeof(UINT32));
849 data->seek(data->tell() - sizeof(UINT32));
850 }
851
852 if (curField != nullptr)
853 {
854 // Note: Two data copies that can potentially be avoided:
855 // - Copy from stream into a temporary buffer (use stream directly for decoding)
856 // - Internally the field will do a value copy of the decoded object (ideally we decode directly into the destination)
857 void* fieldValue = bs_stack_alloc(typeSize);
858 READ_FROM_BUFFER(fieldValue, typeSize)
859
860 curField->fromBuffer(rttiInstance, output.get(), fieldValue);
861 bs_stack_free(fieldValue);
862 }
863 else
864 SKIP_READ(typeSize);
865
866 break;
867 }
868 case SerializableFT_DataBlock:
869 {
870 RTTIManagedDataBlockFieldBase* curField = static_cast<RTTIManagedDataBlockFieldBase*>(curGenericField);
871
872 // Data block size
873 UINT32 dataBlockSize = 0;
874 READ_FROM_BUFFER(&dataBlockSize, DATA_BLOCK_TYPE_FIELD_SIZE)
875
876 // Data block data
877 if (curField != nullptr)
878 {
879 if (data->isFile()) // Allow streaming
880 {
881 const size_t dataBlockOffset = data->tell();
882 curField->setValue(rttiInstance, output.get(), data, dataBlockSize);
883 REPORT_READ(dataBlockSize);
884
885 // Seek past the data (use original offset in case the field read from the stream)
886 data->seek(dataBlockOffset + dataBlockSize);
887 }
888 else
889 {
890 UINT8* dataBlockBuffer = (UINT8*)bs_alloc(dataBlockSize);
891 READ_FROM_BUFFER(dataBlockBuffer, dataBlockSize)
892
893 SPtr<DataStream> stream = bs_shared_ptr_new<MemoryDataStream>(dataBlockBuffer, dataBlockSize);
894 curField->setValue(rttiInstance, output.get(), stream, dataBlockSize);
895 }
896 }
897 else
898 SKIP_READ(dataBlockSize)
899
900 break;
901 }
902 default:
903 BS_EXCEPT(InternalErrorException,
904 "Error decoding data. Encountered a type I don't know how to decode. Type: " + toString(UINT32(fieldType)) +
905 ", Is array: " + toString(isArray));
906 }
907 }
908 }
909
910 finalizeObject(output.get());
911 return false;
912 }
913
914 UINT8* BinarySerializer::complexTypeToBuffer(IReflectable* object, UINT8* buffer, UINT32& bufferLength,
915 UINT32* bytesWritten, std::function<UINT8*(UINT8*, UINT32, UINT32&)> flushBufferCallback, bool shallow)
916 {
917 if (object != nullptr)
918 {
919 buffer = encodeEntry(object, 0, buffer, bufferLength, bytesWritten, flushBufferCallback, shallow);
920
921 // Encode terminator field
922 // Complex types require terminator fields because they can be embedded within other complex types and we need
923 // to know when their fields end and parent's resume
924 int metaData = encodeFieldMetaData(0, 0, false, SerializableFT_Plain, false, true);
925 COPY_TO_BUFFER(&metaData, META_SIZE)
926 }
927
928 return buffer;
929 }
930
931 UINT32 BinarySerializer::encodeFieldMetaData(UINT16 id, UINT8 size, bool array, SerializableFieldType type,
932 bool hasDynamicSize, bool terminator)
933 {
934 // If O == 0 - Meta contains field information (Encoded using this method)
935 //// Encoding: IIII IIII IIII IIII SSSS SSSS xTYP DCAO
936 //// I - Id
937 //// S - Size
938 //// C - Complex
939 //// A - Array
940 //// D - Data block
941 //// P - Complex ptr
942 //// O - Object descriptor
943 //// Y - Plain field has dynamic size
944 //// T - Terminator (last field in an object)
945
946 return (id << 16 | size << 8 |
947 (array ? 0x02 : 0) |
948 ((type == SerializableFT_DataBlock) ? 0x04 : 0) |
949 ((type == SerializableFT_Reflectable) ? 0x08 : 0) |
950 ((type == SerializableFT_ReflectablePtr) ? 0x10 : 0) |
951 (hasDynamicSize ? 0x20 : 0) |
952 (terminator ? 0x40 : 0)); // TODO - Low priority. Technically I could encode this much more tightly, and use var-ints for ID
953 }
954
955 void BinarySerializer::decodeFieldMetaData(UINT32 encodedData, UINT16& id, UINT8& size,
956 bool& array, SerializableFieldType& type, bool& hasDynamicSize, bool& terminator)
957 {
958 if(isObjectMetaData(encodedData))
959 {
960 BS_EXCEPT(InternalErrorException,
961 "Meta data represents an object description but is trying to be decoded as a field descriptor.");
962 }
963
964 terminator = (encodedData & 0x40) != 0;
965 hasDynamicSize = (encodedData & 0x20) != 0;
966
967 if((encodedData & 0x10) != 0)
968 type = SerializableFT_ReflectablePtr;
969 else if((encodedData & 0x08) != 0)
970 type = SerializableFT_Reflectable;
971 else if((encodedData & 0x04) != 0)
972 type = SerializableFT_DataBlock;
973 else
974 type = SerializableFT_Plain;
975
976 array = (encodedData & 0x02) != 0;
977 size = (UINT8)((encodedData >> 8) & 0xFF);
978 id = (UINT16)((encodedData >> 16) & 0xFFFF);
979 }
980
981 BinarySerializer::ObjectMetaData BinarySerializer::encodeObjectMetaData(UINT32 objId, UINT32 objTypeId, bool isBaseClass)
982 {
983 // If O == 1 - Meta contains object instance information (Encoded using encodeObjectMetaData)
984 //// Encoding: SSSS SSSS SSSS SSSS xxxx xxxx xxxx xxBO
985 //// S - Size of the object identifier
986 //// O - Object descriptor
987 //// B - Base class indicator
988
989 if(objId > 1073741823)
990 {
991 BS_EXCEPT(InvalidParametersException, "Object ID is larger than we can store (max 30 bits): " + toString(objId));
992 }
993
994 ObjectMetaData metaData;
995 metaData.objectMeta = (objId << 2) | (isBaseClass ? 0x02 : 0) | 0x01;
996 metaData.typeId = objTypeId;
997 return metaData;
998 }
999
1000 void BinarySerializer::decodeObjectMetaData(ObjectMetaData encodedData, UINT32& objId, UINT32& objTypeId, bool& isBaseClass)
1001 {
1002 if(!isObjectMetaData(encodedData.objectMeta))
1003 {
1004 BS_EXCEPT(InternalErrorException,
1005 "Meta data represents a field description but is trying to be decoded as an object descriptor.");
1006 }
1007
1008 objId = (encodedData.objectMeta >> 2) & 0x3FFFFFFF;
1009 isBaseClass = (encodedData.objectMeta & 0x02) != 0;
1010 objTypeId = encodedData.typeId;
1011 }
1012
1013 bool BinarySerializer::isObjectMetaData(UINT32 encodedData)
1014 {
1015 return ((encodedData & 0x01) != 0);
1016 }
1017
1018 UINT8* BinarySerializer::dataBlockToBuffer(UINT8* data, UINT32 size, UINT8* buffer, UINT32& bufferLength, UINT32* bytesWritten,
1019 std::function<UINT8*(UINT8* buffer, UINT32 bytesWritten, UINT32& newBufferSize)> flushBufferCallback)
1020 {
1021 UINT32 remainingSize = size;
1022 while (remainingSize > 0)
1023 {
1024 UINT32 remainingSpaceInBuffer = bufferLength - *bytesWritten;
1025
1026 if (remainingSize <= remainingSpaceInBuffer)
1027 {
1028 COPY_TO_BUFFER(data, remainingSize);
1029 remainingSize = 0;
1030 }
1031 else
1032 {
1033 memcpy(buffer, data, remainingSpaceInBuffer);
1034 buffer += remainingSpaceInBuffer;
1035 *bytesWritten += remainingSpaceInBuffer;
1036 data += remainingSpaceInBuffer;
1037 remainingSize -= remainingSpaceInBuffer;
1038
1039 mTotalBytesWritten += *bytesWritten;
1040 buffer = flushBufferCallback(buffer - *bytesWritten, *bytesWritten, bufferLength);
1041 if (buffer == nullptr || bufferLength == 0)
1042 return nullptr;
1043
1044 *bytesWritten = 0;
1045 }
1046 }
1047
1048 return buffer;
1049 }
1050
1051 UINT32 BinarySerializer::findOrCreatePersistentId(IReflectable* object)
1052 {
1053 void* ptrAddress = (void*)object;
1054
1055 auto findIter = mObjectAddrToId.find(ptrAddress);
1056 if(findIter != mObjectAddrToId.end())
1057 return findIter->second;
1058
1059 UINT32 objId = mLastUsedObjectId++;
1060 mObjectAddrToId.insert(std::make_pair(ptrAddress, objId));
1061
1062 return objId;
1063 }
1064
1065 UINT32 BinarySerializer::registerObjectPtr(SPtr<IReflectable> object)
1066 {
1067 if(object == nullptr)
1068 return 0;
1069
1070 void* ptrAddress = (void*)object.get();
1071
1072 auto iterFind = mObjectAddrToId.find(ptrAddress);
1073 if(iterFind == mObjectAddrToId.end())
1074 {
1075 UINT32 objId = findOrCreatePersistentId(object.get());
1076
1077 mObjectsToEncode.push_back(ObjectToEncode(objId, object));
1078 mObjectAddrToId.insert(std::make_pair(ptrAddress, objId));
1079
1080 return objId;
1081 }
1082
1083 return iterFind->second;
1084 }
1085}
1086
1087#undef COPY_TO_BUFFER
1088