1/**************************************************************************/
2/* shader_preprocessor.cpp */
3/**************************************************************************/
4/* This file is part of: */
5/* GODOT ENGINE */
6/* https://godotengine.org */
7/**************************************************************************/
8/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
9/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
10/* */
11/* Permission is hereby granted, free of charge, to any person obtaining */
12/* a copy of this software and associated documentation files (the */
13/* "Software"), to deal in the Software without restriction, including */
14/* without limitation the rights to use, copy, modify, merge, publish, */
15/* distribute, sublicense, and/or sell copies of the Software, and to */
16/* permit persons to whom the Software is furnished to do so, subject to */
17/* the following conditions: */
18/* */
19/* The above copyright notice and this permission notice shall be */
20/* included in all copies or substantial portions of the Software. */
21/* */
22/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
23/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
24/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
25/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
26/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
27/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
28/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
29/**************************************************************************/
30
31#include "shader_preprocessor.h"
32#include "core/math/expression.h"
33
34const char32_t CURSOR = 0xFFFF;
35
36// Tokenizer
37
38void ShaderPreprocessor::Tokenizer::add_generated(const ShaderPreprocessor::Token &p_t) {
39 generated.push_back(p_t);
40}
41
42char32_t ShaderPreprocessor::Tokenizer::next() {
43 if (index < size) {
44 return code[index++];
45 }
46 return 0;
47}
48
49int ShaderPreprocessor::Tokenizer::get_line() const {
50 return line;
51}
52
53int ShaderPreprocessor::Tokenizer::get_index() const {
54 return index;
55}
56
57void ShaderPreprocessor::Tokenizer::get_and_clear_generated(LocalVector<char32_t> *r_out) {
58 for (uint32_t i = 0; i < generated.size(); i++) {
59 r_out->push_back(generated[i].text);
60 }
61 generated.clear();
62}
63
64void ShaderPreprocessor::Tokenizer::backtrack(char32_t p_what) {
65 while (index >= 0) {
66 char32_t c = code[index];
67 if (c == p_what) {
68 break;
69 }
70 index--;
71 }
72}
73
74char32_t ShaderPreprocessor::Tokenizer::peek() {
75 if (index < size) {
76 return code[index];
77 }
78 return 0;
79}
80
81int ShaderPreprocessor::Tokenizer::consume_line_continuations(int p_offset) {
82 int skips = 0;
83
84 for (int i = index + p_offset; i < size; i++) {
85 char32_t c = code[i];
86 if (c == '\\') {
87 if (i + 1 < size && code[i + 1] == '\n') {
88 // This line ends with "\" and "\n" continuation.
89 add_generated(Token('\n', line));
90 line++;
91 skips++;
92
93 i = i + 2;
94 index = i;
95 } else {
96 break;
97 }
98 } else if (!is_whitespace(c)) {
99 break;
100 }
101 }
102 return skips;
103}
104
105LocalVector<ShaderPreprocessor::Token> ShaderPreprocessor::Tokenizer::advance(char32_t p_what) {
106 LocalVector<ShaderPreprocessor::Token> tokens;
107
108 while (index < size) {
109 char32_t c = code[index++];
110 if (c == '\\' && consume_line_continuations(-1) > 0) {
111 continue;
112 }
113
114 if (c == '\n') {
115 add_generated(ShaderPreprocessor::Token('\n', line));
116 line++;
117 }
118
119 tokens.push_back(ShaderPreprocessor::Token(c, line));
120
121 if (c == p_what || c == 0) {
122 return tokens;
123 }
124 }
125 return LocalVector<ShaderPreprocessor::Token>();
126}
127
128void ShaderPreprocessor::Tokenizer::skip_whitespace() {
129 while (is_char_space(peek())) {
130 next();
131 }
132}
133
134bool ShaderPreprocessor::Tokenizer::consume_empty_line() {
135 // Read until newline and return true if the content was all whitespace/empty.
136 return tokens_to_string(advance('\n')).strip_edges().size() == 0;
137}
138
139String ShaderPreprocessor::Tokenizer::get_identifier(bool *r_is_cursor, bool p_started) {
140 if (r_is_cursor != nullptr) {
141 *r_is_cursor = false;
142 }
143
144 LocalVector<char32_t> text;
145
146 while (true) {
147 char32_t c = peek();
148 if (c == '\\' && consume_line_continuations(0) > 0) {
149 continue;
150 }
151
152 if (is_char_end(c) || c == '(' || c == ')' || c == ',' || c == ';') {
153 break;
154 }
155
156 if (is_whitespace(c) && p_started) {
157 break;
158 }
159 if (!is_whitespace(c)) {
160 p_started = true;
161 }
162
163 char32_t n = next();
164 if (n == CURSOR) {
165 if (r_is_cursor != nullptr) {
166 *r_is_cursor = true;
167 }
168 } else {
169 if (p_started) {
170 text.push_back(n);
171 }
172 }
173 }
174
175 String id = vector_to_string(text);
176 if (!id.is_valid_identifier()) {
177 return "";
178 }
179
180 return id;
181}
182
183String ShaderPreprocessor::Tokenizer::peek_identifier() {
184 const int original = index;
185 const int original_line = line;
186 String id = get_identifier();
187 index = original;
188 line = original_line;
189 return id;
190}
191
192ShaderPreprocessor::Token ShaderPreprocessor::Tokenizer::get_token() {
193 while (index < size) {
194 const char32_t c = code[index++];
195 const Token t = ShaderPreprocessor::Token(c, line);
196
197 switch (c) {
198 case ' ':
199 case '\t':
200 skip_whitespace();
201 return ShaderPreprocessor::Token(' ', line);
202 case '\n':
203 line++;
204 return t;
205 default:
206 return t;
207 }
208 }
209 return ShaderPreprocessor::Token(char32_t(0), line);
210}
211
212ShaderPreprocessor::Tokenizer::Tokenizer(const String &p_code) {
213 code = p_code;
214 line = 0;
215 index = 0;
216 size = code.size();
217}
218
219// ShaderPreprocessor::CommentRemover
220
221String ShaderPreprocessor::CommentRemover::get_error() const {
222 if (comments_open != 0) {
223 return "Block comment mismatch";
224 }
225 return "";
226}
227
228int ShaderPreprocessor::CommentRemover::get_error_line() const {
229 if (comments_open != 0) {
230 return comment_line_open;
231 }
232 return -1;
233}
234
235char32_t ShaderPreprocessor::CommentRemover::peek() const {
236 if (index < code.size()) {
237 return code[index];
238 }
239 return 0;
240}
241
242bool ShaderPreprocessor::CommentRemover::advance(char32_t p_what) {
243 while (index < code.size()) {
244 char32_t c = code[index++];
245
246 if (c == '\n') {
247 line++;
248 stripped.push_back('\n');
249 }
250
251 if (c == p_what) {
252 return true;
253 }
254 }
255 return false;
256}
257
258String ShaderPreprocessor::CommentRemover::strip() {
259 stripped.clear();
260 index = 0;
261 line = 0;
262 comment_line_open = 0;
263 comments_open = 0;
264 strings_open = 0;
265
266 while (index < code.size()) {
267 char32_t c = code[index++];
268
269 if (c == CURSOR) {
270 // Cursor. Maintain.
271 stripped.push_back(c);
272 } else if (c == '"') {
273 if (strings_open <= 0) {
274 strings_open++;
275 } else {
276 strings_open--;
277 }
278 stripped.push_back(c);
279 } else if (c == '/' && strings_open == 0) {
280 char32_t p = peek();
281 if (p == '/') { // Single line comment.
282 advance('\n');
283 } else if (p == '*') { // Start of a block comment.
284 index++;
285 comment_line_open = line;
286 comments_open++;
287 while (advance('*')) {
288 if (peek() == '/') { // End of a block comment.
289 comments_open--;
290 index++;
291 break;
292 }
293 }
294 } else {
295 stripped.push_back(c);
296 }
297 } else if (c == '*' && strings_open == 0) {
298 if (peek() == '/') { // Unmatched end of a block comment.
299 comment_line_open = line;
300 comments_open--;
301 } else {
302 stripped.push_back(c);
303 }
304 } else if (c == '\n') {
305 line++;
306 stripped.push_back(c);
307 } else {
308 stripped.push_back(c);
309 }
310 }
311 return vector_to_string(stripped);
312}
313
314ShaderPreprocessor::CommentRemover::CommentRemover(const String &p_code) {
315 code = p_code;
316 index = 0;
317 line = 0;
318 comment_line_open = 0;
319 comments_open = 0;
320 strings_open = 0;
321}
322
323// ShaderPreprocessor::Token
324
325ShaderPreprocessor::Token::Token() {
326 text = 0;
327 line = -1;
328}
329
330ShaderPreprocessor::Token::Token(char32_t p_text, int p_line) {
331 text = p_text;
332 line = p_line;
333}
334
335// ShaderPreprocessor
336
337bool ShaderPreprocessor::is_char_word(char32_t p_char) {
338 if ((p_char >= '0' && p_char <= '9') ||
339 (p_char >= 'a' && p_char <= 'z') ||
340 (p_char >= 'A' && p_char <= 'Z') ||
341 p_char == '_') {
342 return true;
343 }
344
345 return false;
346}
347
348bool ShaderPreprocessor::is_char_space(char32_t p_char) {
349 return p_char == ' ' || p_char == '\t';
350}
351
352bool ShaderPreprocessor::is_char_end(char32_t p_char) {
353 return p_char == '\n' || p_char == 0;
354}
355
356String ShaderPreprocessor::vector_to_string(const LocalVector<char32_t> &p_v, int p_start, int p_end) {
357 const int stop = (p_end == -1) ? p_v.size() : p_end;
358 const int count = stop - p_start;
359
360 String result;
361 result.resize(count + 1);
362 for (int i = 0; i < count; i++) {
363 result[i] = p_v[p_start + i];
364 }
365 result[count] = 0; // Ensure string is null terminated for length() to work.
366 return result;
367}
368
369String ShaderPreprocessor::tokens_to_string(const LocalVector<Token> &p_tokens) {
370 LocalVector<char32_t> result;
371 for (const Token &token : p_tokens) {
372 result.push_back(token.text);
373 }
374 return vector_to_string(result);
375}
376
377void ShaderPreprocessor::process_directive(Tokenizer *p_tokenizer) {
378 bool is_cursor;
379 String directive = p_tokenizer->get_identifier(&is_cursor, true);
380 if (is_cursor) {
381 state->completion_type = COMPLETION_TYPE_DIRECTIVE;
382 }
383
384 if (directive == "if") {
385 process_if(p_tokenizer);
386 } else if (directive == "ifdef") {
387 process_ifdef(p_tokenizer);
388 } else if (directive == "ifndef") {
389 process_ifndef(p_tokenizer);
390 } else if (directive == "elif") {
391 process_elif(p_tokenizer);
392 } else if (directive == "else") {
393 process_else(p_tokenizer);
394 } else if (directive == "endif") {
395 process_endif(p_tokenizer);
396 } else if (directive == "define") {
397 process_define(p_tokenizer);
398 } else if (directive == "undef") {
399 process_undef(p_tokenizer);
400 } else if (directive == "include") {
401 process_include(p_tokenizer);
402 } else if (directive == "pragma") {
403 process_pragma(p_tokenizer);
404 } else {
405 set_error(RTR("Unknown directive."), p_tokenizer->get_line());
406 }
407}
408
409void ShaderPreprocessor::process_define(Tokenizer *p_tokenizer) {
410 const int line = p_tokenizer->get_line();
411
412 String label = p_tokenizer->get_identifier();
413 if (label.is_empty()) {
414 set_error(RTR("Invalid macro name."), line);
415 return;
416 }
417
418 if (state->defines.has(label)) {
419 set_error(RTR("Macro redefinition."), line);
420 return;
421 }
422
423 Vector<String> args;
424 if (p_tokenizer->peek() == '(') {
425 // Macro has arguments.
426 p_tokenizer->get_token();
427
428 while (true) {
429 String name = p_tokenizer->get_identifier();
430 if (name.is_empty()) {
431 set_error(RTR("Invalid argument name."), line);
432 return;
433 }
434 args.push_back(name);
435
436 p_tokenizer->skip_whitespace();
437 char32_t next = p_tokenizer->get_token().text;
438 if (next == ')') {
439 break;
440 } else if (next != ',') {
441 set_error(RTR("Expected a comma in the macro argument list."), line);
442 return;
443 }
444 }
445 }
446
447 String body = tokens_to_string(p_tokenizer->advance('\n')).strip_edges();
448 if (body.begins_with("##")) {
449 set_error(RTR("'##' must not appear at beginning of macro expansion."), line);
450 return;
451 }
452 if (body.ends_with("##")) {
453 set_error(RTR("'##' must not appear at end of macro expansion."), line);
454 return;
455 }
456
457 Define *define = memnew(Define);
458 if (!args.is_empty()) {
459 define->arguments = args;
460 }
461 define->body = body;
462 state->defines[label] = define;
463}
464
465void ShaderPreprocessor::process_elif(Tokenizer *p_tokenizer) {
466 const int line = p_tokenizer->get_line();
467
468 if (state->current_branch == nullptr || state->current_branch->else_defined) {
469 set_error(RTR("Unmatched elif."), line);
470 return;
471 }
472 if (state->previous_region != nullptr) {
473 state->previous_region->to_line = line - 1;
474 }
475
476 String body = tokens_to_string(p_tokenizer->advance('\n')).strip_edges();
477 if (body.is_empty()) {
478 set_error(RTR("Missing condition."), line);
479 return;
480 }
481
482 Error error = expand_condition(body, line, body);
483 if (error != OK) {
484 return;
485 }
486
487 error = expand_macros(body, line, body);
488 if (error != OK) {
489 return;
490 }
491
492 Expression expression;
493 Vector<String> names;
494 error = expression.parse(body, names);
495 if (error != OK) {
496 set_error(expression.get_error_text(), line);
497 return;
498 }
499
500 Variant v = expression.execute(Array(), nullptr, false);
501 if (v.get_type() == Variant::NIL) {
502 set_error(RTR("Condition evaluation error."), line);
503 return;
504 }
505
506 bool skip = false;
507 for (int i = 0; i < state->current_branch->conditions.size(); i++) {
508 if (state->current_branch->conditions[i]) {
509 skip = true;
510 break;
511 }
512 }
513
514 bool success = !skip && v.booleanize();
515 start_branch_condition(p_tokenizer, success, true);
516
517 if (state->save_regions) {
518 add_region(line + 1, success, state->previous_region->parent);
519 }
520}
521
522void ShaderPreprocessor::process_else(Tokenizer *p_tokenizer) {
523 const int line = p_tokenizer->get_line();
524
525 if (state->current_branch == nullptr || state->current_branch->else_defined) {
526 set_error(RTR("Unmatched else."), line);
527 return;
528 }
529 if (state->previous_region != nullptr) {
530 state->previous_region->to_line = line - 1;
531 }
532
533 if (!p_tokenizer->consume_empty_line()) {
534 set_error(RTR("Invalid else."), p_tokenizer->get_line());
535 }
536
537 bool skip = false;
538 for (int i = 0; i < state->current_branch->conditions.size(); i++) {
539 if (state->current_branch->conditions[i]) {
540 skip = true;
541 break;
542 }
543 }
544 state->current_branch->else_defined = true;
545
546 if (state->save_regions) {
547 add_region(line + 1, !skip, state->previous_region->parent);
548 }
549
550 if (skip) {
551 Vector<String> ends;
552 ends.push_back("endif");
553 next_directive(p_tokenizer, ends);
554 }
555}
556
557void ShaderPreprocessor::process_endif(Tokenizer *p_tokenizer) {
558 const int line = p_tokenizer->get_line();
559
560 state->condition_depth--;
561 if (state->condition_depth < 0) {
562 set_error(RTR("Unmatched endif."), line);
563 return;
564 }
565 if (state->previous_region != nullptr) {
566 state->previous_region->to_line = line - 1;
567 state->previous_region = state->previous_region->parent;
568 }
569
570 if (!p_tokenizer->consume_empty_line()) {
571 set_error(RTR("Invalid endif."), line);
572 }
573
574 state->current_branch = state->current_branch->parent;
575 state->branches.pop_back();
576}
577
578void ShaderPreprocessor::process_if(Tokenizer *p_tokenizer) {
579 const int line = p_tokenizer->get_line();
580
581 String body = tokens_to_string(p_tokenizer->advance('\n')).strip_edges();
582 if (body.is_empty()) {
583 set_error(RTR("Missing condition."), line);
584 return;
585 }
586
587 Error error = expand_condition(body, line, body);
588 if (error != OK) {
589 return;
590 }
591
592 error = expand_macros(body, line, body);
593 if (error != OK) {
594 return;
595 }
596
597 Expression expression;
598 Vector<String> names;
599 error = expression.parse(body, names);
600 if (error != OK) {
601 set_error(expression.get_error_text(), line);
602 return;
603 }
604
605 Variant v = expression.execute(Array(), nullptr, false);
606 if (v.get_type() == Variant::NIL) {
607 set_error(RTR("Condition evaluation error."), line);
608 return;
609 }
610
611 bool success = v.booleanize();
612 start_branch_condition(p_tokenizer, success);
613
614 if (state->save_regions) {
615 add_region(line + 1, success, state->previous_region);
616 }
617}
618
619void ShaderPreprocessor::process_ifdef(Tokenizer *p_tokenizer) {
620 const int line = p_tokenizer->get_line();
621
622 String label = p_tokenizer->get_identifier();
623 if (label.is_empty()) {
624 set_error(RTR("Invalid macro name."), line);
625 return;
626 }
627
628 if (!p_tokenizer->consume_empty_line()) {
629 set_error(RTR("Invalid ifdef."), line);
630 return;
631 }
632
633 bool success = state->defines.has(label);
634 start_branch_condition(p_tokenizer, success);
635
636 if (state->save_regions) {
637 add_region(line + 1, success, state->previous_region);
638 }
639}
640
641void ShaderPreprocessor::process_ifndef(Tokenizer *p_tokenizer) {
642 const int line = p_tokenizer->get_line();
643
644 String label = p_tokenizer->get_identifier();
645 if (label.is_empty()) {
646 set_error(RTR("Invalid macro name."), line);
647 return;
648 }
649
650 if (!p_tokenizer->consume_empty_line()) {
651 set_error(RTR("Invalid ifndef."), line);
652 return;
653 }
654
655 bool success = !state->defines.has(label);
656 start_branch_condition(p_tokenizer, success);
657
658 if (state->save_regions) {
659 add_region(line + 1, success, state->previous_region);
660 }
661}
662
663void ShaderPreprocessor::process_include(Tokenizer *p_tokenizer) {
664 const int line = p_tokenizer->get_line();
665
666 p_tokenizer->advance('"');
667 String path = tokens_to_string(p_tokenizer->advance('"'));
668 for (int i = 0; i < path.length(); i++) {
669 if (path[i] == '\n') {
670 break; //stop parsing
671 }
672 if (path[i] == CURSOR) {
673 state->completion_type = COMPLETION_TYPE_INCLUDE_PATH;
674 break;
675 }
676 }
677 path = path.substr(0, path.length() - 1);
678
679 if (path.is_empty() || !p_tokenizer->consume_empty_line()) {
680 set_error(RTR("Invalid path."), line);
681 return;
682 }
683
684 path = path.simplify_path();
685 if (path.is_relative_path()) {
686 path = state->current_filename.get_base_dir().path_join(path);
687 }
688
689 if (!ResourceLoader::exists(path)) {
690 set_error(RTR("Shader include file does not exist:") + " " + path, line);
691 return;
692 }
693
694 Ref<Resource> res = ResourceLoader::load(path);
695 if (res.is_null()) {
696 set_error(RTR("Shader include load failed. Does the shader include exist? Is there a cyclic dependency?"), line);
697 return;
698 }
699
700 Ref<ShaderInclude> shader_inc = res;
701 if (shader_inc.is_null()) {
702 set_error(RTR("Shader include resource type is wrong."), line);
703 return;
704 }
705
706 String included = shader_inc->get_code();
707 if (!included.is_empty()) {
708 uint64_t code_hash = included.hash64();
709 if (state->cyclic_include_hashes.find(code_hash)) {
710 set_error(RTR("Cyclic include found") + ": " + path, line);
711 return;
712 }
713 }
714
715 state->shader_includes.insert(shader_inc);
716
717 const String real_path = shader_inc->get_path();
718 if (state->includes.has(real_path)) {
719 // Already included, skip.
720 // This is a valid check because 2 separate include paths could use some
721 // of the same shared functions from a common shader include.
722 return;
723 }
724
725 // Mark as included.
726 state->includes.insert(real_path);
727
728 state->include_depth++;
729 if (state->include_depth > 25) {
730 set_error(RTR("Shader max include depth exceeded."), line);
731 return;
732 }
733
734 String old_filename = state->current_filename;
735 state->current_filename = real_path;
736 ShaderPreprocessor processor;
737
738 int prev_condition_depth = state->condition_depth;
739 state->condition_depth = 0;
740
741 FilePosition fp;
742 fp.file = state->current_filename;
743 fp.line = line + 1;
744 state->include_positions.push_back(fp);
745
746 String result;
747 processor.preprocess(state, included, result);
748 add_to_output("@@>" + real_path + "\n"); // Add token for enter include path
749 add_to_output(result);
750 add_to_output("\n@@<\n"); // Add token for exit include path
751
752 // Reset to last include if there are no errors. We want to use this as context.
753 if (state->error.is_empty()) {
754 state->current_filename = old_filename;
755 state->include_positions.pop_back();
756 } else {
757 return;
758 }
759
760 state->include_depth--;
761 state->condition_depth = prev_condition_depth;
762}
763
764void ShaderPreprocessor::process_pragma(Tokenizer *p_tokenizer) {
765 const int line = p_tokenizer->get_line();
766
767 bool is_cursor;
768 const String label = p_tokenizer->get_identifier(&is_cursor);
769 if (is_cursor) {
770 state->completion_type = COMPLETION_TYPE_PRAGMA;
771 }
772
773 if (label.is_empty()) {
774 set_error(RTR("Invalid pragma directive."), line);
775 return;
776 }
777
778 // Rxplicitly handle pragma values here.
779 // If more pragma options are created, then refactor into a more defined structure.
780 if (label == "disable_preprocessor") {
781 state->disabled = true;
782 } else {
783 set_error(RTR("Invalid pragma directive."), line);
784 return;
785 }
786
787 if (!p_tokenizer->consume_empty_line()) {
788 set_error(RTR("Invalid pragma directive."), line);
789 return;
790 }
791}
792
793void ShaderPreprocessor::process_undef(Tokenizer *p_tokenizer) {
794 const int line = p_tokenizer->get_line();
795 const String label = p_tokenizer->get_identifier();
796 if (label.is_empty() || !p_tokenizer->consume_empty_line()) {
797 set_error(RTR("Invalid undef."), line);
798 return;
799 }
800
801 if (state->defines.has(label)) {
802 memdelete(state->defines[label]);
803 state->defines.erase(label);
804 }
805}
806
807void ShaderPreprocessor::add_region(int p_line, bool p_enabled, Region *p_parent_region) {
808 Region region;
809 region.file = state->current_filename;
810 region.enabled = p_enabled;
811 region.from_line = p_line;
812 region.parent = p_parent_region;
813 state->previous_region = &state->regions[region.file].push_back(region)->get();
814}
815
816void ShaderPreprocessor::start_branch_condition(Tokenizer *p_tokenizer, bool p_success, bool p_continue) {
817 if (!p_continue) {
818 state->condition_depth++;
819 state->current_branch = &state->branches.push_back(Branch(p_success, state->current_branch))->get();
820 } else {
821 state->current_branch->conditions.push_back(p_success);
822 }
823 if (!p_success) {
824 Vector<String> ends;
825 ends.push_back("elif");
826 ends.push_back("else");
827 ends.push_back("endif");
828 next_directive(p_tokenizer, ends);
829 }
830}
831
832void ShaderPreprocessor::expand_output_macros(int p_start, int p_line_number) {
833 String line = vector_to_string(output, p_start, output.size());
834
835 Error error = expand_macros(line, p_line_number - 1, line); // We are already on next line, so -1.
836 if (error != OK) {
837 return;
838 }
839
840 output.resize(p_start);
841
842 add_to_output(line);
843}
844
845Error ShaderPreprocessor::expand_condition(const String &p_string, int p_line, String &r_expanded) {
846 // Checks bracket count to be even + check the cursor position.
847 {
848 int bracket_start_count = 0;
849 int bracket_end_count = 0;
850
851 for (int i = 0; i < p_string.size(); i++) {
852 switch (p_string[i]) {
853 case CURSOR:
854 state->completion_type = COMPLETION_TYPE_CONDITION;
855 break;
856 case '(':
857 bracket_start_count++;
858 break;
859 case ')':
860 bracket_end_count++;
861 break;
862 }
863 }
864 if (bracket_start_count > bracket_end_count) {
865 _set_expected_error(")", p_line);
866 return FAILED;
867 }
868 if (bracket_end_count > bracket_start_count) {
869 _set_expected_error("(", p_line);
870 return FAILED;
871 }
872 }
873
874 String result = p_string;
875
876 int index = 0;
877 int index_start = 0;
878 int index_end = 0;
879
880 while (find_match(result, "defined", index, index_start)) {
881 bool open_bracket = false;
882 bool found_word = false;
883 bool word_completed = false;
884
885 LocalVector<char32_t> text;
886 int post_bracket_index = -1;
887 int size = result.size();
888
889 for (int i = (index_start - 1); i < size; i++) {
890 char32_t c = result[i];
891 if (c == 0) {
892 if (found_word) {
893 word_completed = true;
894 }
895 break;
896 }
897 char32_t cs[] = { c, '\0' };
898 String s = String(cs);
899 bool is_space = is_char_space(c);
900
901 if (word_completed) {
902 if (c == ')') {
903 continue;
904 }
905 if (c == '|' || c == '&') {
906 if (open_bracket) {
907 _set_unexpected_token_error(s, p_line);
908 return FAILED;
909 }
910 break;
911 } else if (!is_space) {
912 _set_unexpected_token_error(s, p_line);
913 return FAILED;
914 }
915 } else if (is_space) {
916 if (found_word && !open_bracket) {
917 index_end = i;
918 word_completed = true;
919 }
920 } else if (c == '(') {
921 if (open_bracket) {
922 _set_unexpected_token_error(s, p_line);
923 return FAILED;
924 }
925 open_bracket = true;
926 } else if (c == ')') {
927 if (open_bracket) {
928 if (!found_word) {
929 _set_unexpected_token_error(s, p_line);
930 return FAILED;
931 }
932 open_bracket = false;
933 post_bracket_index = i + 1;
934 } else {
935 index_end = i;
936 }
937 word_completed = true;
938 } else if (is_char_word(c)) {
939 text.push_back(c);
940 found_word = true;
941 } else {
942 _set_unexpected_token_error(s, p_line);
943 return FAILED;
944 }
945 }
946
947 if (word_completed) {
948 if (open_bracket) {
949 _set_expected_error(")", p_line);
950 return FAILED;
951 }
952 if (post_bracket_index != -1) {
953 index_end = post_bracket_index;
954 }
955
956 String body = state->defines.has(vector_to_string(text)) ? "true" : "false";
957 String temp = result;
958
959 result = result.substr(0, index) + body;
960 index_start = result.length();
961 if (index_end > 0) {
962 result += temp.substr(index_end);
963 }
964 } else {
965 set_error(RTR("Invalid macro name."), p_line);
966 return FAILED;
967 }
968 }
969 r_expanded = result;
970 return OK;
971}
972
973Error ShaderPreprocessor::expand_macros(const String &p_string, int p_line, String &r_expanded) {
974 String iterative = p_string;
975 int pass_count = 0;
976 bool expanded = true;
977
978 while (expanded) {
979 expanded = false;
980
981 // As long as we find something to expand, keep going.
982 for (const RBMap<String, Define *>::Element *E = state->defines.front(); E; E = E->next()) {
983 if (expand_macros_once(iterative, p_line, E, iterative)) {
984 expanded = true;
985 }
986 }
987
988 pass_count++;
989 if (pass_count > 50) {
990 set_error(RTR("Macro expansion limit exceeded."), p_line);
991 break;
992 }
993 }
994
995 r_expanded = iterative;
996
997 if (!state->error.is_empty()) {
998 return FAILED;
999 }
1000 return OK;
1001}
1002
1003bool ShaderPreprocessor::expand_macros_once(const String &p_line, int p_line_number, const RBMap<String, Define *>::Element *p_define_pair, String &r_expanded) {
1004 String result = p_line;
1005
1006 const String &key = p_define_pair->key();
1007 const Define *define = p_define_pair->value();
1008
1009 int index_start = 0;
1010 int index = 0;
1011 if (find_match(result, key, index, index_start)) {
1012 String body = define->body;
1013 if (define->arguments.size() > 0) {
1014 // Complex macro with arguments.
1015
1016 int args_start = -1;
1017 int args_end = -1;
1018 int brackets_open = 0;
1019 Vector<String> args;
1020 for (int i = index_start - 1; i < p_line.length(); i++) {
1021 bool add_argument = false;
1022 bool reached_end = false;
1023 char32_t c = p_line[i];
1024
1025 if (c == '(') {
1026 brackets_open++;
1027 if (brackets_open == 1) {
1028 args_start = i + 1;
1029 args_end = -1;
1030 }
1031 } else if (c == ')') {
1032 brackets_open--;
1033 if (brackets_open == 0) {
1034 args_end = i;
1035 add_argument = true;
1036 reached_end = true;
1037 }
1038 } else if (c == ',') {
1039 if (brackets_open == 1) {
1040 args_end = i;
1041 add_argument = true;
1042 }
1043 }
1044
1045 if (add_argument) {
1046 if (args_start == -1 || args_end == -1) {
1047 set_error(RTR("Invalid macro argument list."), p_line_number);
1048 return false;
1049 }
1050
1051 String arg = p_line.substr(args_start, args_end - args_start).strip_edges();
1052 if (arg.is_empty()) {
1053 set_error(RTR("Invalid macro argument."), p_line_number);
1054 return false;
1055 }
1056 args.append(arg);
1057
1058 args_start = args_end + 1;
1059 }
1060
1061 if (reached_end) {
1062 break;
1063 }
1064 }
1065
1066 if (args.size() != define->arguments.size()) {
1067 set_error(RTR("Invalid macro argument count."), p_line_number);
1068 return false;
1069 }
1070
1071 // Insert macro arguments into the body.
1072 for (int i = 0; i < args.size(); i++) {
1073 String arg_name = define->arguments[i];
1074 int arg_index_start = 0;
1075 int arg_index = 0;
1076 while (find_match(body, arg_name, arg_index, arg_index_start)) {
1077 body = body.substr(0, arg_index) + args[i] + body.substr(arg_index + arg_name.length(), body.length() - (arg_index + arg_name.length()));
1078 // Manually reset arg_index_start to where the arg value of the define finishes.
1079 // This ensures we don't skip the other args of this macro in the string.
1080 arg_index_start = arg_index + args[i].length() + 1;
1081 }
1082 }
1083
1084 concatenate_macro_body(body);
1085
1086 result = result.substr(0, index) + " " + body + " " + result.substr(args_end + 1, result.length());
1087 } else {
1088 concatenate_macro_body(body);
1089
1090 result = result.substr(0, index) + " " + body + " " + result.substr(index + key.length(), result.length() - (index + key.length()));
1091 }
1092
1093 r_expanded = result;
1094 return true;
1095 }
1096
1097 return false;
1098}
1099
1100bool ShaderPreprocessor::find_match(const String &p_string, const String &p_value, int &r_index, int &r_index_start) {
1101 // Looks for value in string and then determines if the boundaries
1102 // are non-word characters. This method semi-emulates \b in regex.
1103 r_index = p_string.find(p_value, r_index_start);
1104 while (r_index > -1) {
1105 if (r_index > 0) {
1106 if (is_char_word(p_string[r_index - 1])) {
1107 r_index_start = r_index + 1;
1108 r_index = p_string.find(p_value, r_index_start);
1109 continue;
1110 }
1111 }
1112
1113 if (r_index + p_value.length() < p_string.length()) {
1114 if (is_char_word(p_string[r_index + p_value.length()])) {
1115 r_index_start = r_index + p_value.length() + 1;
1116 r_index = p_string.find(p_value, r_index_start);
1117 continue;
1118 }
1119 }
1120
1121 // Return and shift index start automatically for next call.
1122 r_index_start = r_index + p_value.length() + 1;
1123 return true;
1124 }
1125
1126 return false;
1127}
1128
1129void ShaderPreprocessor::concatenate_macro_body(String &r_body) {
1130 int index_start = r_body.find("##");
1131 while (index_start > -1) {
1132 int index_end = index_start + 2; // First character after ##.
1133 // The macro was checked during creation so this should never happen.
1134 ERR_FAIL_INDEX(index_end, r_body.size());
1135
1136 // If there more than two # in a row, then it's not a concatenation.
1137 bool is_concat = true;
1138 while (index_end <= r_body.length() && r_body[index_end] == '#') {
1139 index_end++;
1140 is_concat = false;
1141 }
1142 if (!is_concat) {
1143 index_start = r_body.find("##", index_end);
1144 continue;
1145 }
1146
1147 // Skip whitespace after ##.
1148 while (index_end < r_body.length() && is_char_space(r_body[index_end])) {
1149 index_end++;
1150 }
1151
1152 // Skip whitespace before ##.
1153 while (index_start >= 1 && is_char_space(r_body[index_start - 1])) {
1154 index_start--;
1155 }
1156
1157 r_body = r_body.substr(0, index_start) + r_body.substr(index_end, r_body.length() - index_end);
1158
1159 index_start = r_body.find("##", index_start);
1160 }
1161}
1162
1163String ShaderPreprocessor::next_directive(Tokenizer *p_tokenizer, const Vector<String> &p_directives) {
1164 const int line = p_tokenizer->get_line();
1165 int nesting = 0;
1166
1167 while (true) {
1168 p_tokenizer->advance('#');
1169
1170 String id = p_tokenizer->peek_identifier();
1171 if (id.is_empty()) {
1172 break;
1173 }
1174
1175 if (nesting == 0) {
1176 for (int i = 0; i < p_directives.size(); i++) {
1177 if (p_directives[i] == id) {
1178 p_tokenizer->backtrack('#');
1179 return id;
1180 }
1181 }
1182 }
1183
1184 if (id == "ifdef" || id == "ifndef" || id == "if") {
1185 nesting++;
1186 } else if (id == "endif") {
1187 nesting--;
1188 }
1189 }
1190
1191 set_error(RTR("Can't find matching branch directive."), line);
1192 return "";
1193}
1194
1195void ShaderPreprocessor::add_to_output(const String &p_str) {
1196 for (int i = 0; i < p_str.length(); i++) {
1197 output.push_back(p_str[i]);
1198 }
1199}
1200
1201void ShaderPreprocessor::set_error(const String &p_error, int p_line) {
1202 if (state->error.is_empty()) {
1203 state->error = p_error;
1204 FilePosition fp;
1205 fp.file = state->current_filename;
1206 fp.line = p_line + 1;
1207 state->include_positions.push_back(fp);
1208 }
1209}
1210
1211ShaderPreprocessor::Define *ShaderPreprocessor::create_define(const String &p_body) {
1212 ShaderPreprocessor::Define *define = memnew(Define);
1213 define->body = p_body;
1214 return define;
1215}
1216
1217void ShaderPreprocessor::clear_state() {
1218 if (state != nullptr) {
1219 for (const RBMap<String, Define *>::Element *E = state->defines.front(); E; E = E->next()) {
1220 memdelete(E->get());
1221 }
1222 state->defines.clear();
1223 }
1224 state = nullptr;
1225}
1226
1227Error ShaderPreprocessor::preprocess(State *p_state, const String &p_code, String &r_result) {
1228 output.clear();
1229
1230 state = p_state;
1231
1232 CommentRemover remover(p_code);
1233 String stripped = remover.strip();
1234 String error = remover.get_error();
1235 if (!error.is_empty()) {
1236 set_error(error, remover.get_error_line());
1237 return FAILED;
1238 }
1239
1240 // Track code hashes to prevent cyclic include.
1241 uint64_t code_hash = p_code.hash64();
1242 state->cyclic_include_hashes.push_back(code_hash);
1243
1244 Tokenizer p_tokenizer(stripped);
1245 int last_size = 0;
1246 bool has_symbols_before_directive = false;
1247
1248 while (true) {
1249 const Token &t = p_tokenizer.get_token();
1250
1251 if (t.text == 0) {
1252 break;
1253 }
1254
1255 // Add autogenerated tokens if there are any.
1256 p_tokenizer.get_and_clear_generated(&output);
1257
1258 if (state->disabled) {
1259 // Preprocessor was disabled.
1260 // Read the rest of the file into the output.
1261 output.push_back(t.text);
1262 continue;
1263 }
1264
1265 if (t.text == '#') {
1266 if (has_symbols_before_directive) {
1267 set_error(RTR("Invalid symbols placed before directive."), p_tokenizer.get_line());
1268 state->cyclic_include_hashes.erase(code_hash); // Remove this hash.
1269 return FAILED;
1270 }
1271 process_directive(&p_tokenizer);
1272 } else {
1273 if (is_char_end(t.text)) {
1274 expand_output_macros(last_size, p_tokenizer.get_line());
1275 last_size = output.size();
1276 has_symbols_before_directive = false;
1277 } else if (!is_char_space(t.text)) {
1278 has_symbols_before_directive = true;
1279 }
1280 output.push_back(t.text);
1281 }
1282
1283 if (!state->error.is_empty()) {
1284 state->cyclic_include_hashes.erase(code_hash); // Remove this hash.
1285 return FAILED;
1286 }
1287 }
1288 state->cyclic_include_hashes.erase(code_hash); // Remove this hash.
1289
1290 if (!state->disabled) {
1291 if (state->condition_depth != 0) {
1292 set_error(RTR("Unmatched conditional statement."), p_tokenizer.line);
1293 return FAILED;
1294 }
1295
1296 expand_output_macros(last_size, p_tokenizer.get_line());
1297 }
1298
1299 r_result = vector_to_string(output);
1300
1301 return OK;
1302}
1303
1304Error ShaderPreprocessor::preprocess(const String &p_code, const String &p_filename, String &r_result, String *r_error_text, List<FilePosition> *r_error_position, List<Region> *r_regions, HashSet<Ref<ShaderInclude>> *r_includes, List<ScriptLanguage::CodeCompletionOption> *r_completion_options, List<ScriptLanguage::CodeCompletionOption> *r_completion_defines, IncludeCompletionFunction p_include_completion_func) {
1305 State pp_state;
1306 if (!p_filename.is_empty()) {
1307 pp_state.current_filename = p_filename;
1308 pp_state.save_regions = r_regions != nullptr;
1309 }
1310 Error err = preprocess(&pp_state, p_code, r_result);
1311 if (err != OK) {
1312 if (r_error_text) {
1313 *r_error_text = pp_state.error;
1314 }
1315 if (r_error_position) {
1316 *r_error_position = pp_state.include_positions;
1317 }
1318 }
1319 if (r_regions) {
1320 *r_regions = pp_state.regions[p_filename];
1321 }
1322 if (r_includes) {
1323 *r_includes = pp_state.shader_includes;
1324 }
1325
1326 if (r_completion_defines) {
1327 for (const KeyValue<String, Define *> &E : state->defines) {
1328 ScriptLanguage::CodeCompletionOption option(E.key, ScriptLanguage::CODE_COMPLETION_KIND_CONSTANT);
1329 r_completion_defines->push_back(option);
1330 }
1331 }
1332
1333 if (r_completion_options) {
1334 switch (pp_state.completion_type) {
1335 case COMPLETION_TYPE_DIRECTIVE: {
1336 List<String> options;
1337 get_keyword_list(&options, true, true);
1338
1339 for (const String &E : options) {
1340 ScriptLanguage::CodeCompletionOption option(E, ScriptLanguage::CODE_COMPLETION_KIND_PLAIN_TEXT);
1341 r_completion_options->push_back(option);
1342 }
1343
1344 } break;
1345 case COMPLETION_TYPE_PRAGMA: {
1346 List<String> options;
1347 ShaderPreprocessor::get_pragma_list(&options);
1348
1349 for (const String &E : options) {
1350 ScriptLanguage::CodeCompletionOption option(E, ScriptLanguage::CODE_COMPLETION_KIND_PLAIN_TEXT);
1351 r_completion_options->push_back(option);
1352 }
1353
1354 } break;
1355 case COMPLETION_TYPE_CONDITION: {
1356 ScriptLanguage::CodeCompletionOption option("defined", ScriptLanguage::CODE_COMPLETION_KIND_PLAIN_TEXT);
1357 r_completion_options->push_back(option);
1358
1359 } break;
1360 case COMPLETION_TYPE_INCLUDE_PATH: {
1361 if (p_include_completion_func && r_completion_options) {
1362 p_include_completion_func(r_completion_options);
1363 }
1364
1365 } break;
1366 default: {
1367 }
1368 }
1369 }
1370
1371 clear_state();
1372
1373 return err;
1374}
1375
1376void ShaderPreprocessor::get_keyword_list(List<String> *r_keywords, bool p_include_shader_keywords, bool p_ignore_context_keywords) {
1377 r_keywords->push_back("define");
1378 if (!p_ignore_context_keywords) {
1379 r_keywords->push_back("defined");
1380 }
1381 r_keywords->push_back("elif");
1382 if (p_include_shader_keywords) {
1383 r_keywords->push_back("else");
1384 }
1385 r_keywords->push_back("endif");
1386 if (p_include_shader_keywords) {
1387 r_keywords->push_back("if");
1388 }
1389 r_keywords->push_back("ifdef");
1390 r_keywords->push_back("ifndef");
1391 r_keywords->push_back("include");
1392 r_keywords->push_back("pragma");
1393 r_keywords->push_back("undef");
1394}
1395
1396void ShaderPreprocessor::get_pragma_list(List<String> *r_pragmas) {
1397 r_pragmas->push_back("disable_preprocessor");
1398}
1399
1400ShaderPreprocessor::ShaderPreprocessor() {
1401}
1402
1403ShaderPreprocessor::~ShaderPreprocessor() {
1404}
1405