1/**************************************************************************/
2/* translation_loader_po.cpp */
3/**************************************************************************/
4/* This file is part of: */
5/* GODOT ENGINE */
6/* https://godotengine.org */
7/**************************************************************************/
8/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
9/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
10/* */
11/* Permission is hereby granted, free of charge, to any person obtaining */
12/* a copy of this software and associated documentation files (the */
13/* "Software"), to deal in the Software without restriction, including */
14/* without limitation the rights to use, copy, modify, merge, publish, */
15/* distribute, sublicense, and/or sell copies of the Software, and to */
16/* permit persons to whom the Software is furnished to do so, subject to */
17/* the following conditions: */
18/* */
19/* The above copyright notice and this permission notice shall be */
20/* included in all copies or substantial portions of the Software. */
21/* */
22/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
23/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
24/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
25/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
26/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
27/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
28/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
29/**************************************************************************/
30
31#include "translation_loader_po.h"
32
33#include "core/io/file_access.h"
34#include "core/string/translation.h"
35#include "core/string/translation_po.h"
36
37Ref<Resource> TranslationLoaderPO::load_translation(Ref<FileAccess> f, Error *r_error) {
38 if (r_error) {
39 *r_error = ERR_FILE_CORRUPT;
40 }
41
42 const String path = f->get_path();
43 Ref<TranslationPO> translation = Ref<TranslationPO>(memnew(TranslationPO));
44 String config;
45
46 uint32_t magic = f->get_32();
47 if (magic == 0x950412de) {
48 // Load binary MO file.
49
50 uint16_t version_maj = f->get_16();
51 uint16_t version_min = f->get_16();
52 ERR_FAIL_COND_V_MSG(version_maj > 1, Ref<Resource>(), vformat("Unsupported MO file %s, version %d.%d.", path, version_maj, version_min));
53
54 uint32_t num_strings = f->get_32();
55 uint32_t id_table_offset = f->get_32();
56 uint32_t trans_table_offset = f->get_32();
57
58 // Read string tables.
59 for (uint32_t i = 0; i < num_strings; i++) {
60 String msg_id;
61 String msg_id_plural;
62 String msg_context;
63
64 // Read id strings and context.
65 {
66 Vector<uint8_t> data;
67 f->seek(id_table_offset + i * 8);
68 uint32_t str_start = 0;
69 uint32_t str_len = f->get_32();
70 uint32_t str_offset = f->get_32();
71
72 data.resize(str_len + 1);
73 f->seek(str_offset);
74 f->get_buffer(data.ptrw(), str_len);
75 data.write[str_len] = 0;
76
77 bool is_plural = false;
78 for (uint32_t j = 0; j < str_len + 1; j++) {
79 if (data[j] == 0x04) {
80 msg_context.parse_utf8((const char *)data.ptr(), j);
81 str_start = j + 1;
82 }
83 if (data[j] == 0x00) {
84 if (is_plural) {
85 msg_id_plural.parse_utf8((const char *)(data.ptr() + str_start), j - str_start);
86 } else {
87 msg_id.parse_utf8((const char *)(data.ptr() + str_start), j - str_start);
88 is_plural = true;
89 }
90 str_start = j + 1;
91 }
92 }
93 }
94
95 // Read translated strings.
96 {
97 Vector<uint8_t> data;
98 f->seek(trans_table_offset + i * 8);
99 uint32_t str_len = f->get_32();
100 uint32_t str_offset = f->get_32();
101
102 data.resize(str_len + 1);
103 f->seek(str_offset);
104 f->get_buffer(data.ptrw(), str_len);
105 data.write[str_len] = 0;
106
107 if (msg_id.is_empty()) {
108 config = String::utf8((const char *)data.ptr(), str_len);
109 // Record plural rule.
110 int p_start = config.find("Plural-Forms");
111 if (p_start != -1) {
112 int p_end = config.find("\n", p_start);
113 translation->set_plural_rule(config.substr(p_start, p_end - p_start));
114 }
115 } else {
116 uint32_t str_start = 0;
117 Vector<String> plural_msg;
118 for (uint32_t j = 0; j < str_len + 1; j++) {
119 if (data[j] == 0x00) {
120 if (msg_id_plural.is_empty()) {
121 translation->add_message(msg_id, String::utf8((const char *)(data.ptr() + str_start), j - str_start), msg_context);
122 } else {
123 plural_msg.push_back(String::utf8((const char *)(data.ptr() + str_start), j - str_start));
124 }
125 str_start = j + 1;
126 }
127 }
128 if (!plural_msg.is_empty()) {
129 translation->add_plural_message(msg_id, plural_msg, msg_context);
130 }
131 }
132 }
133 }
134
135 } else {
136 // Try to load as text PO file.
137 f->seek(0);
138
139 enum Status {
140 STATUS_NONE,
141 STATUS_READING_ID,
142 STATUS_READING_STRING,
143 STATUS_READING_CONTEXT,
144 STATUS_READING_PLURAL,
145 };
146
147 Status status = STATUS_NONE;
148
149 String msg_id;
150 String msg_str;
151 String msg_context;
152 Vector<String> msgs_plural;
153
154 if (r_error) {
155 *r_error = ERR_FILE_CORRUPT;
156 }
157
158 int line = 1;
159 int plural_forms = 0;
160 int plural_index = -1;
161 bool entered_context = false;
162 bool skip_this = false;
163 bool skip_next = false;
164 bool is_eof = false;
165
166 while (!is_eof) {
167 String l = f->get_line().strip_edges();
168 is_eof = f->eof_reached();
169
170 // If we reached last line and it's not a content line, break, otherwise let processing that last loop
171 if (is_eof && l.is_empty()) {
172 if (status == STATUS_READING_ID || status == STATUS_READING_CONTEXT || (status == STATUS_READING_PLURAL && plural_index != plural_forms - 1)) {
173 ERR_FAIL_V_MSG(Ref<Resource>(), "Unexpected EOF while reading PO file at: " + path + ":" + itos(line));
174 } else {
175 break;
176 }
177 }
178
179 if (l.begins_with("msgctxt")) {
180 ERR_FAIL_COND_V_MSG(status != STATUS_READING_STRING && status != STATUS_READING_PLURAL, Ref<Resource>(), "Unexpected 'msgctxt', was expecting 'msgid_plural' or 'msgstr' before 'msgctxt' while parsing: " + path + ":" + itos(line));
181
182 // In PO file, "msgctxt" appears before "msgid". If we encounter a "msgctxt", we add what we have read
183 // and set "entered_context" to true to prevent adding twice.
184 if (!skip_this && !msg_id.is_empty()) {
185 if (status == STATUS_READING_STRING) {
186 translation->add_message(msg_id, msg_str, msg_context);
187 } else if (status == STATUS_READING_PLURAL) {
188 ERR_FAIL_COND_V_MSG(plural_index != plural_forms - 1, Ref<Resource>(), "Number of 'msgstr[]' doesn't match with number of plural forms: " + path + ":" + itos(line));
189 translation->add_plural_message(msg_id, msgs_plural, msg_context);
190 }
191 }
192 msg_context = "";
193 l = l.substr(7, l.length()).strip_edges();
194 status = STATUS_READING_CONTEXT;
195 entered_context = true;
196 }
197
198 if (l.begins_with("msgid_plural")) {
199 if (plural_forms == 0) {
200 ERR_FAIL_V_MSG(Ref<Resource>(), "PO file uses 'msgid_plural' but 'Plural-Forms' is invalid or missing in header: " + path + ":" + itos(line));
201 } else if (status != STATUS_READING_ID) {
202 ERR_FAIL_V_MSG(Ref<Resource>(), "Unexpected 'msgid_plural', was expecting 'msgid' before 'msgid_plural' while parsing: " + path + ":" + itos(line));
203 }
204 // We don't record the message in "msgid_plural" itself as tr_n(), TTRN(), RTRN() interfaces provide the plural string already.
205 // We just have to reset variables related to plurals for "msgstr[]" later on.
206 l = l.substr(12, l.length()).strip_edges();
207 plural_index = -1;
208 msgs_plural.clear();
209 msgs_plural.resize(plural_forms);
210 status = STATUS_READING_PLURAL;
211 } else if (l.begins_with("msgid")) {
212 ERR_FAIL_COND_V_MSG(status == STATUS_READING_ID, Ref<Resource>(), "Unexpected 'msgid', was expecting 'msgstr' while parsing: " + path + ":" + itos(line));
213
214 if (!msg_id.is_empty()) {
215 if (!skip_this && !entered_context) {
216 if (status == STATUS_READING_STRING) {
217 translation->add_message(msg_id, msg_str, msg_context);
218 } else if (status == STATUS_READING_PLURAL) {
219 ERR_FAIL_COND_V_MSG(plural_index != plural_forms - 1, Ref<Resource>(), "Number of 'msgstr[]' doesn't match with number of plural forms: " + path + ":" + itos(line));
220 translation->add_plural_message(msg_id, msgs_plural, msg_context);
221 }
222 }
223 } else if (config.is_empty()) {
224 config = msg_str;
225 // Record plural rule.
226 int p_start = config.find("Plural-Forms");
227 if (p_start != -1) {
228 int p_end = config.find("\n", p_start);
229 translation->set_plural_rule(config.substr(p_start, p_end - p_start));
230 plural_forms = translation->get_plural_forms();
231 }
232 }
233
234 l = l.substr(5, l.length()).strip_edges();
235 status = STATUS_READING_ID;
236 // If we did not encounter msgctxt, we reset context to empty to reset it.
237 if (!entered_context) {
238 msg_context = "";
239 }
240 msg_id = "";
241 msg_str = "";
242 skip_this = skip_next;
243 skip_next = false;
244 entered_context = false;
245 }
246
247 if (l.begins_with("msgstr[")) {
248 ERR_FAIL_COND_V_MSG(status != STATUS_READING_PLURAL, Ref<Resource>(), "Unexpected 'msgstr[]', was expecting 'msgid_plural' before 'msgstr[]' while parsing: " + path + ":" + itos(line));
249 plural_index++; // Increment to add to the next slot in vector msgs_plural.
250 l = l.substr(9, l.length()).strip_edges();
251 } else if (l.begins_with("msgstr")) {
252 ERR_FAIL_COND_V_MSG(status != STATUS_READING_ID, Ref<Resource>(), "Unexpected 'msgstr', was expecting 'msgid' before 'msgstr' while parsing: " + path + ":" + itos(line));
253 l = l.substr(6, l.length()).strip_edges();
254 status = STATUS_READING_STRING;
255 }
256
257 if (l.is_empty() || l.begins_with("#")) {
258 if (l.contains("fuzzy")) {
259 skip_next = true;
260 }
261 line++;
262 continue; // Nothing to read or comment.
263 }
264
265 ERR_FAIL_COND_V_MSG(!l.begins_with("\"") || status == STATUS_NONE, Ref<Resource>(), "Invalid line '" + l + "' while parsing: " + path + ":" + itos(line));
266
267 l = l.substr(1, l.length());
268 // Find final quote, ignoring escaped ones (\").
269 // The escape_next logic is necessary to properly parse things like \\"
270 // where the backslash is the one being escaped, not the quote.
271 int end_pos = -1;
272 bool escape_next = false;
273 for (int i = 0; i < l.length(); i++) {
274 if (l[i] == '\\' && !escape_next) {
275 escape_next = true;
276 continue;
277 }
278
279 if (l[i] == '"' && !escape_next) {
280 end_pos = i;
281 break;
282 }
283
284 escape_next = false;
285 }
286
287 ERR_FAIL_COND_V_MSG(end_pos == -1, Ref<Resource>(), "Expected '\"' at end of message while parsing: " + path + ":" + itos(line));
288
289 l = l.substr(0, end_pos);
290 l = l.c_unescape();
291
292 if (status == STATUS_READING_ID) {
293 msg_id += l;
294 } else if (status == STATUS_READING_STRING) {
295 msg_str += l;
296 } else if (status == STATUS_READING_CONTEXT) {
297 msg_context += l;
298 } else if (status == STATUS_READING_PLURAL && plural_index >= 0) {
299 ERR_FAIL_COND_V_MSG(plural_index >= plural_forms, Ref<Resource>(), "Unexpected plural form while parsing: " + path + ":" + itos(line));
300 msgs_plural.write[plural_index] = msgs_plural[plural_index] + l;
301 }
302
303 line++;
304 }
305
306 // Add the last set of data from last iteration.
307 if (status == STATUS_READING_STRING) {
308 if (!msg_id.is_empty()) {
309 if (!skip_this) {
310 translation->add_message(msg_id, msg_str, msg_context);
311 }
312 } else if (config.is_empty()) {
313 config = msg_str;
314 }
315 } else if (status == STATUS_READING_PLURAL) {
316 if (!skip_this && !msg_id.is_empty()) {
317 ERR_FAIL_COND_V_MSG(plural_index != plural_forms - 1, Ref<Resource>(), "Number of 'msgstr[]' doesn't match with number of plural forms: " + path + ":" + itos(line));
318 translation->add_plural_message(msg_id, msgs_plural, msg_context);
319 }
320 }
321 }
322
323 ERR_FAIL_COND_V_MSG(config.is_empty(), Ref<Resource>(), "No config found in file: " + path + ".");
324
325 Vector<String> configs = config.split("\n");
326 for (int i = 0; i < configs.size(); i++) {
327 String c = configs[i].strip_edges();
328 int p = c.find(":");
329 if (p == -1) {
330 continue;
331 }
332 String prop = c.substr(0, p).strip_edges();
333 String value = c.substr(p + 1, c.length()).strip_edges();
334
335 if (prop == "X-Language" || prop == "Language") {
336 translation->set_locale(value);
337 }
338 }
339
340 if (r_error) {
341 *r_error = OK;
342 }
343
344 return translation;
345}
346
347Ref<Resource> TranslationLoaderPO::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress, CacheMode p_cache_mode) {
348 if (r_error) {
349 *r_error = ERR_CANT_OPEN;
350 }
351
352 Ref<FileAccess> f = FileAccess::open(p_path, FileAccess::READ);
353 ERR_FAIL_COND_V_MSG(f.is_null(), Ref<Resource>(), "Cannot open file '" + p_path + "'.");
354
355 return load_translation(f, r_error);
356}
357
358void TranslationLoaderPO::get_recognized_extensions(List<String> *p_extensions) const {
359 p_extensions->push_back("po");
360 p_extensions->push_back("mo");
361}
362
363bool TranslationLoaderPO::handles_type(const String &p_type) const {
364 return (p_type == "Translation");
365}
366
367String TranslationLoaderPO::get_resource_type(const String &p_path) const {
368 if (p_path.get_extension().to_lower() == "po" || p_path.get_extension().to_lower() == "mo") {
369 return "Translation";
370 }
371 return "";
372}
373