1// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2// for details. All rights reserved. Use of this source code is governed by a
3// BSD-style license that can be found in the LICENSE file.
4
5#include "platform/unicode.h"
6
7#include "vm/allocation.h"
8#include "vm/globals.h"
9#include "vm/object.h"
10
11namespace dart {
12
13// A constant mask that can be 'and'ed with a word of data to determine if it
14// is all ASCII (with no Latin1 characters).
15#if defined(ARCH_IS_64_BIT)
16static const uintptr_t kAsciiWordMask = DART_UINT64_C(0x8080808080808080);
17#else
18static const uintptr_t kAsciiWordMask = 0x80808080u;
19#endif
20
21intptr_t Utf8::Length(const String& str) {
22 if (str.IsOneByteString() || str.IsExternalOneByteString()) {
23 // For 1-byte strings, all code points < 0x80 have single-byte UTF-8
24 // encodings and all >= 0x80 have two-byte encodings. To get the length,
25 // start with the number of code points and add the number of high bits in
26 // the bytes.
27 uintptr_t char_length = str.Length();
28 uintptr_t length = char_length;
29 const uintptr_t* data;
30 NoSafepointScope no_safepoint;
31 if (str.IsOneByteString()) {
32 data = reinterpret_cast<const uintptr_t*>(OneByteString::DataStart(str));
33 } else {
34 data = reinterpret_cast<const uintptr_t*>(
35 ExternalOneByteString::DataStart(str));
36 }
37 uintptr_t i;
38 for (i = sizeof(uintptr_t); i <= char_length; i += sizeof(uintptr_t)) {
39 uintptr_t chunk = *data++;
40 chunk &= kAsciiWordMask;
41 if (chunk != 0) {
42// Shuffle the bits until we have a count of bits in the low nibble.
43#if defined(ARCH_IS_64_BIT)
44 chunk += chunk >> 32;
45#endif
46 chunk += chunk >> 16;
47 chunk += chunk >> 8;
48 length += (chunk >> 7) & 0xf;
49 }
50 }
51 // Take care of the tail of the string, the last length % wordsize chars.
52 i -= sizeof(uintptr_t);
53 for (; i < char_length; i++) {
54 if (str.CharAt(i) > kMaxOneByteChar) length++;
55 }
56 return length;
57 }
58
59 // Slow case for 2-byte strings that handles surrogate pairs and longer UTF-8
60 // encodings.
61 intptr_t length = 0;
62 String::CodePointIterator it(str);
63 while (it.Next()) {
64 int32_t ch = it.Current();
65 length += Utf8::Length(ch);
66 }
67 return length;
68}
69
70intptr_t Utf8::Encode(const String& src, char* dst, intptr_t len) {
71 uintptr_t array_len = len;
72 intptr_t pos = 0;
73 ASSERT(static_cast<intptr_t>(array_len) >= Length(src));
74 if (src.IsOneByteString() || src.IsExternalOneByteString()) {
75 // For 1-byte strings, all code points < 0x80 have single-byte UTF-8
76 // encodings and all >= 0x80 have two-byte encodings.
77 const uintptr_t* data;
78 NoSafepointScope scope;
79 if (src.IsOneByteString()) {
80 data = reinterpret_cast<const uintptr_t*>(OneByteString::DataStart(src));
81 } else {
82 data = reinterpret_cast<const uintptr_t*>(
83 ExternalOneByteString::DataStart(src));
84 }
85 uintptr_t char_length = src.Length();
86 uintptr_t pos = 0;
87 ASSERT(kMaxOneByteChar + 1 == 0x80);
88 for (uintptr_t i = 0; i < char_length; i += sizeof(uintptr_t)) {
89 // Read the input one word at a time and just write it verbatim if it is
90 // plain ASCII, as determined by the mask.
91 if (i + sizeof(uintptr_t) <= char_length &&
92 (*data & kAsciiWordMask) == 0 &&
93 pos + sizeof(uintptr_t) <= array_len) {
94 StoreUnaligned(reinterpret_cast<uintptr_t*>(dst + pos), *data);
95 pos += sizeof(uintptr_t);
96 } else {
97 // Process up to one word of input that contains non-ASCII Latin1
98 // characters.
99 const uint8_t* p = reinterpret_cast<const uint8_t*>(data);
100 const uint8_t* limit =
101 Utils::Minimum(p + sizeof(uintptr_t), p + (char_length - i));
102 for (; p < limit; p++) {
103 uint8_t c = *p;
104 // These calls to Length and Encode get inlined and the cases for 3
105 // and 4 byte sequences are removed.
106 intptr_t bytes = Length(c);
107 if (pos + bytes > array_len) {
108 return pos;
109 }
110 Encode(c, reinterpret_cast<char*>(dst) + pos);
111 pos += bytes;
112 }
113 }
114 data++;
115 }
116 } else {
117 // For two-byte strings, which can contain 3 and 4-byte UTF-8 encodings,
118 // which can result in surrogate pairs, use the more general code.
119 String::CodePointIterator it(src);
120 while (it.Next()) {
121 int32_t ch = it.Current();
122 ASSERT(!Utf::IsOutOfRange(ch));
123 if (Utf16::IsSurrogate(ch)) {
124 // Encode unpaired surrogates as replacement characters to ensure the
125 // output is valid UTF-8. Encoded size is the same (3), so the computed
126 // length is still valid.
127 ch = Utf::kReplacementChar;
128 }
129 intptr_t num_bytes = Utf8::Length(ch);
130 if (pos + num_bytes > len) {
131 break;
132 }
133 Utf8::Encode(ch, &dst[pos]);
134 pos += num_bytes;
135 }
136 }
137 return pos;
138}
139
140} // namespace dart
141