1/**************************************************************************/
2/* optimized_translation.cpp */
3/**************************************************************************/
4/* This file is part of: */
5/* GODOT ENGINE */
6/* https://godotengine.org */
7/**************************************************************************/
8/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
9/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
10/* */
11/* Permission is hereby granted, free of charge, to any person obtaining */
12/* a copy of this software and associated documentation files (the */
13/* "Software"), to deal in the Software without restriction, including */
14/* without limitation the rights to use, copy, modify, merge, publish, */
15/* distribute, sublicense, and/or sell copies of the Software, and to */
16/* permit persons to whom the Software is furnished to do so, subject to */
17/* the following conditions: */
18/* */
19/* The above copyright notice and this permission notice shall be */
20/* included in all copies or substantial portions of the Software. */
21/* */
22/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
23/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
24/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
25/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
26/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
27/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
28/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
29/**************************************************************************/
30
31#include "optimized_translation.h"
32
33#include "core/templates/pair.h"
34
35extern "C" {
36#include "thirdparty/misc/smaz.h"
37}
38
39struct CompressedString {
40 int orig_len = 0;
41 CharString compressed;
42 int offset = 0;
43};
44
45void OptimizedTranslation::generate(const Ref<Translation> &p_from) {
46 // This method compresses a Translation instance.
47 // Right now, it doesn't handle context or plurals, so Translation subclasses using plurals or context (i.e TranslationPO) shouldn't be compressed.
48#ifdef TOOLS_ENABLED
49 ERR_FAIL_COND(p_from.is_null());
50 List<StringName> keys;
51 p_from->get_message_list(&keys);
52
53 int size = Math::larger_prime(keys.size());
54
55 Vector<Vector<Pair<int, CharString>>> buckets;
56 Vector<HashMap<uint32_t, int>> table;
57 Vector<uint32_t> hfunc_table;
58 Vector<CompressedString> compressed;
59
60 table.resize(size);
61 hfunc_table.resize(size);
62 buckets.resize(size);
63 compressed.resize(keys.size());
64
65 int idx = 0;
66 int total_compression_size = 0;
67
68 for (const StringName &E : keys) {
69 //hash string
70 CharString cs = E.operator String().utf8();
71 uint32_t h = hash(0, cs.get_data());
72 Pair<int, CharString> p;
73 p.first = idx;
74 p.second = cs;
75 buckets.write[h % size].push_back(p);
76
77 //compress string
78 CharString src_s = p_from->get_message(E).operator String().utf8();
79 CompressedString ps;
80 ps.orig_len = src_s.size();
81 ps.offset = total_compression_size;
82
83 if (ps.orig_len != 0) {
84 CharString dst_s;
85 dst_s.resize(src_s.size());
86 int ret = smaz_compress(src_s.get_data(), src_s.size(), dst_s.ptrw(), src_s.size());
87 if (ret >= src_s.size()) {
88 //if compressed is larger than original, just use original
89 ps.orig_len = src_s.size();
90 ps.compressed = src_s;
91 } else {
92 dst_s.resize(ret);
93 //ps.orig_len=;
94 ps.compressed = dst_s;
95 }
96 } else {
97 ps.orig_len = 1;
98 ps.compressed.resize(1);
99 ps.compressed[0] = 0;
100 }
101
102 compressed.write[idx] = ps;
103 total_compression_size += ps.compressed.size();
104 idx++;
105 }
106
107 int bucket_table_size = 0;
108
109 for (int i = 0; i < size; i++) {
110 const Vector<Pair<int, CharString>> &b = buckets[i];
111 HashMap<uint32_t, int> &t = table.write[i];
112
113 if (b.size() == 0) {
114 continue;
115 }
116
117 int d = 1;
118 int item = 0;
119
120 while (item < b.size()) {
121 uint32_t slot = hash(d, b[item].second.get_data());
122 if (t.has(slot)) {
123 item = 0;
124 d++;
125 t.clear();
126 } else {
127 t[slot] = b[item].first;
128 item++;
129 }
130 }
131
132 hfunc_table.write[i] = d;
133 bucket_table_size += 2 + b.size() * 4;
134 }
135
136 ERR_FAIL_COND(bucket_table_size == 0);
137
138 hash_table.resize(size);
139 bucket_table.resize(bucket_table_size);
140
141 int *htwb = hash_table.ptrw();
142 int *btwb = bucket_table.ptrw();
143
144 uint32_t *htw = (uint32_t *)&htwb[0];
145 uint32_t *btw = (uint32_t *)&btwb[0];
146
147 int btindex = 0;
148
149 for (int i = 0; i < size; i++) {
150 const HashMap<uint32_t, int> &t = table[i];
151 if (t.size() == 0) {
152 htw[i] = 0xFFFFFFFF; //nothing
153 continue;
154 }
155
156 htw[i] = btindex;
157 btw[btindex++] = t.size();
158 btw[btindex++] = hfunc_table[i];
159
160 for (const KeyValue<uint32_t, int> &E : t) {
161 btw[btindex++] = E.key;
162 btw[btindex++] = compressed[E.value].offset;
163 btw[btindex++] = compressed[E.value].compressed.size();
164 btw[btindex++] = compressed[E.value].orig_len;
165 }
166 }
167
168 strings.resize(total_compression_size);
169 uint8_t *cw = strings.ptrw();
170
171 for (int i = 0; i < compressed.size(); i++) {
172 memcpy(&cw[compressed[i].offset], compressed[i].compressed.get_data(), compressed[i].compressed.size());
173 }
174
175 ERR_FAIL_COND(btindex != bucket_table_size);
176 set_locale(p_from->get_locale());
177
178#endif
179}
180
181bool OptimizedTranslation::_set(const StringName &p_name, const Variant &p_value) {
182 String prop_name = p_name.operator String();
183 if (prop_name == "hash_table") {
184 hash_table = p_value;
185 } else if (prop_name == "bucket_table") {
186 bucket_table = p_value;
187 } else if (prop_name == "strings") {
188 strings = p_value;
189 } else if (prop_name == "load_from") {
190 generate(p_value);
191 } else {
192 return false;
193 }
194
195 return true;
196}
197
198bool OptimizedTranslation::_get(const StringName &p_name, Variant &r_ret) const {
199 String prop_name = p_name.operator String();
200 if (prop_name == "hash_table") {
201 r_ret = hash_table;
202 } else if (prop_name == "bucket_table") {
203 r_ret = bucket_table;
204 } else if (prop_name == "strings") {
205 r_ret = strings;
206 } else {
207 return false;
208 }
209
210 return true;
211}
212
213StringName OptimizedTranslation::get_message(const StringName &p_src_text, const StringName &p_context) const {
214 // p_context passed in is ignore. The use of context is not yet supported in OptimizedTranslation.
215
216 int htsize = hash_table.size();
217
218 if (htsize == 0) {
219 return StringName();
220 }
221
222 CharString str = p_src_text.operator String().utf8();
223 uint32_t h = hash(0, str.get_data());
224
225 const int *htr = hash_table.ptr();
226 const uint32_t *htptr = (const uint32_t *)&htr[0];
227 const int *btr = bucket_table.ptr();
228 const uint32_t *btptr = (const uint32_t *)&btr[0];
229 const uint8_t *sr = strings.ptr();
230 const char *sptr = (const char *)&sr[0];
231
232 uint32_t p = htptr[h % htsize];
233
234 if (p == 0xFFFFFFFF) {
235 return StringName(); //nothing
236 }
237
238 const Bucket &bucket = *(const Bucket *)&btptr[p];
239
240 h = hash(bucket.func, str.get_data());
241
242 int idx = -1;
243
244 for (int i = 0; i < bucket.size; i++) {
245 if (bucket.elem[i].key == h) {
246 idx = i;
247 break;
248 }
249 }
250
251 if (idx == -1) {
252 return StringName();
253 }
254
255 if (bucket.elem[idx].comp_size == bucket.elem[idx].uncomp_size) {
256 String rstr;
257 rstr.parse_utf8(&sptr[bucket.elem[idx].str_offset], bucket.elem[idx].uncomp_size);
258
259 return rstr;
260 } else {
261 CharString uncomp;
262 uncomp.resize(bucket.elem[idx].uncomp_size + 1);
263 smaz_decompress(&sptr[bucket.elem[idx].str_offset], bucket.elem[idx].comp_size, uncomp.ptrw(), bucket.elem[idx].uncomp_size);
264 String rstr;
265 rstr.parse_utf8(uncomp.get_data());
266 return rstr;
267 }
268}
269
270Vector<String> OptimizedTranslation::get_translated_message_list() const {
271 Vector<String> msgs;
272
273 const int *htr = hash_table.ptr();
274 const uint32_t *htptr = (const uint32_t *)&htr[0];
275 const int *btr = bucket_table.ptr();
276 const uint32_t *btptr = (const uint32_t *)&btr[0];
277 const uint8_t *sr = strings.ptr();
278 const char *sptr = (const char *)&sr[0];
279
280 for (int i = 0; i < hash_table.size(); i++) {
281 uint32_t p = htptr[i];
282 if (p != 0xFFFFFFFF) {
283 const Bucket &bucket = *(const Bucket *)&btptr[p];
284 for (int j = 0; j < bucket.size; j++) {
285 if (bucket.elem[j].comp_size == bucket.elem[j].uncomp_size) {
286 String rstr;
287 rstr.parse_utf8(&sptr[bucket.elem[j].str_offset], bucket.elem[j].uncomp_size);
288 msgs.push_back(rstr);
289 } else {
290 CharString uncomp;
291 uncomp.resize(bucket.elem[j].uncomp_size + 1);
292 smaz_decompress(&sptr[bucket.elem[j].str_offset], bucket.elem[j].comp_size, uncomp.ptrw(), bucket.elem[j].uncomp_size);
293 String rstr;
294 rstr.parse_utf8(uncomp.get_data());
295 msgs.push_back(rstr);
296 }
297 }
298 }
299 }
300 return msgs;
301}
302
303StringName OptimizedTranslation::get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context) const {
304 // The use of plurals translation is not yet supported in OptimizedTranslation.
305 return get_message(p_src_text, p_context);
306}
307
308void OptimizedTranslation::_get_property_list(List<PropertyInfo> *p_list) const {
309 p_list->push_back(PropertyInfo(Variant::PACKED_INT32_ARRAY, "hash_table"));
310 p_list->push_back(PropertyInfo(Variant::PACKED_INT32_ARRAY, "bucket_table"));
311 p_list->push_back(PropertyInfo(Variant::PACKED_BYTE_ARRAY, "strings"));
312 p_list->push_back(PropertyInfo(Variant::OBJECT, "load_from", PROPERTY_HINT_RESOURCE_TYPE, "Translation", PROPERTY_USAGE_EDITOR));
313}
314
315void OptimizedTranslation::_bind_methods() {
316 ClassDB::bind_method(D_METHOD("generate", "from"), &OptimizedTranslation::generate);
317}
318