1 | // Scintilla source code edit control |
2 | /** @file LexCaml.cxx |
3 | ** Lexer for Objective Caml. |
4 | **/ |
5 | // Copyright 2005-2009 by Robert Roessler <robertr@rftp.com> |
6 | // The License.txt file describes the conditions under which this software may be distributed. |
7 | /* Release History |
8 | 20050204 Initial release. |
9 | 20050205 Quick compiler standards/"cleanliness" adjustment. |
10 | 20050206 Added cast for IsLeadByte(). |
11 | 20050209 Changes to "external" build support. |
12 | 20050306 Fix for 1st-char-in-doc "corner" case. |
13 | 20050502 Fix for [harmless] one-past-the-end coloring. |
14 | 20050515 Refined numeric token recognition logic. |
15 | 20051125 Added 2nd "optional" keywords class. |
16 | 20051129 Support "magic" (read-only) comments for RCaml. |
17 | 20051204 Swtich to using StyleContext infrastructure. |
18 | 20090629 Add full Standard ML '97 support. |
19 | */ |
20 | |
21 | #include <stdlib.h> |
22 | #include <string.h> |
23 | #include <stdio.h> |
24 | #include <stdarg.h> |
25 | #include <assert.h> |
26 | #include <ctype.h> |
27 | |
28 | #include <string> |
29 | #include <string_view> |
30 | |
31 | #include "ILexer.h" |
32 | #include "Scintilla.h" |
33 | #include "SciLexer.h" |
34 | |
35 | #include "WordList.h" |
36 | #include "LexAccessor.h" |
37 | #include "Accessor.h" |
38 | #include "StyleContext.h" |
39 | #include "CharacterSet.h" |
40 | #include "LexerModule.h" |
41 | |
42 | #if defined(__clang__) |
43 | #pragma clang diagnostic ignored "-Wcomma" |
44 | #endif |
45 | |
46 | // Since the Microsoft __iscsym[f] funcs are not ANSI... |
47 | inline int iscaml(int c) {return isalnum(c) || c == '_';} |
48 | inline int iscamlf(int c) {return isalpha(c) || c == '_';} |
49 | |
50 | static const int baseT[24] = { |
51 | 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* A - L */ |
52 | 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0,16 /* M - X */ |
53 | }; |
54 | |
55 | using namespace Lexilla; |
56 | |
57 | static void ColouriseCamlDoc( |
58 | Sci_PositionU startPos, Sci_Position length, |
59 | int initStyle, |
60 | WordList *keywordlists[], |
61 | Accessor &styler) |
62 | { |
63 | // initialize styler |
64 | StyleContext sc(startPos, length, initStyle, styler); |
65 | |
66 | Sci_PositionU chToken = 0; |
67 | int chBase = 0, chLit = 0; |
68 | WordList& keywords = *keywordlists[0]; |
69 | WordList& keywords2 = *keywordlists[1]; |
70 | WordList& keywords3 = *keywordlists[2]; |
71 | const bool isSML = keywords.InList("andalso" ); |
72 | const int useMagic = styler.GetPropertyInt("lexer.caml.magic" , 0); |
73 | |
74 | // set up [initial] state info (terminating states that shouldn't "bleed") |
75 | const int state_ = sc.state & 0x0f; |
76 | if (state_ <= SCE_CAML_CHAR |
77 | || (isSML && state_ == SCE_CAML_STRING)) |
78 | sc.state = SCE_CAML_DEFAULT; |
79 | int nesting = (state_ >= SCE_CAML_COMMENT)? (state_ - SCE_CAML_COMMENT): 0; |
80 | |
81 | // foreach char in range... |
82 | while (sc.More()) { |
83 | // set up [per-char] state info |
84 | int state2 = -1; // (ASSUME no state change) |
85 | Sci_Position chColor = sc.currentPos - 1;// (ASSUME standard coloring range) |
86 | bool advance = true; // (ASSUME scanner "eats" 1 char) |
87 | |
88 | // step state machine |
89 | switch (sc.state & 0x0f) { |
90 | case SCE_CAML_DEFAULT: |
91 | chToken = sc.currentPos; // save [possible] token start (JIC) |
92 | // it's wide open; what do we have? |
93 | if (iscamlf(sc.ch)) |
94 | state2 = SCE_CAML_IDENTIFIER; |
95 | else if (!isSML && sc.Match('`') && iscamlf(sc.chNext)) |
96 | state2 = SCE_CAML_TAGNAME; |
97 | else if (!isSML && sc.Match('#') && isdigit(sc.chNext)) |
98 | state2 = SCE_CAML_LINENUM; |
99 | else if (isdigit(sc.ch)) { |
100 | // it's a number, assume base 10 |
101 | state2 = SCE_CAML_NUMBER, chBase = 10; |
102 | if (sc.Match('0')) { |
103 | // there MAY be a base specified... |
104 | const char* baseC = "bBoOxX" ; |
105 | if (isSML) { |
106 | if (sc.chNext == 'w') |
107 | sc.Forward(); // (consume SML "word" indicator) |
108 | baseC = "x" ; |
109 | } |
110 | // ... change to specified base AS REQUIRED |
111 | if (strchr(baseC, sc.chNext)) |
112 | chBase = baseT[tolower(sc.chNext) - 'a'], sc.Forward(); |
113 | } |
114 | } else if (!isSML && sc.Match('\'')) // (Caml char literal?) |
115 | state2 = SCE_CAML_CHAR, chLit = 0; |
116 | else if (isSML && sc.Match('#', '"')) // (SML char literal?) |
117 | state2 = SCE_CAML_CHAR, sc.Forward(); |
118 | else if (sc.Match('"')) |
119 | state2 = SCE_CAML_STRING; |
120 | else if (sc.Match('(', '*')) |
121 | state2 = SCE_CAML_COMMENT, sc.Forward(), sc.ch = ' '; // (*)... |
122 | else if (strchr("!?~" /* Caml "prefix-symbol" */ |
123 | "=<>@^|&+-*/$%" /* Caml "infix-symbol" */ |
124 | "()[]{};,:.#" , sc.ch) // Caml "bracket" or ;,:.# |
125 | // SML "extra" ident chars |
126 | || (isSML && (sc.Match('\\') || sc.Match('`')))) |
127 | state2 = SCE_CAML_OPERATOR; |
128 | break; |
129 | |
130 | case SCE_CAML_IDENTIFIER: |
131 | // [try to] interpret as [additional] identifier char |
132 | if (!(iscaml(sc.ch) || sc.Match('\''))) { |
133 | const Sci_Position n = sc.currentPos - chToken; |
134 | if (n < 24) { |
135 | // length is believable as keyword, [re-]construct token |
136 | char t[24]; |
137 | for (Sci_Position i = -n; i < 0; i++) |
138 | t[n + i] = static_cast<char>(sc.GetRelative(i)); |
139 | t[n] = '\0'; |
140 | // special-case "_" token as KEYWORD |
141 | if ((n == 1 && sc.chPrev == '_') || keywords.InList(t)) |
142 | sc.ChangeState(SCE_CAML_KEYWORD); |
143 | else if (keywords2.InList(t)) |
144 | sc.ChangeState(SCE_CAML_KEYWORD2); |
145 | else if (keywords3.InList(t)) |
146 | sc.ChangeState(SCE_CAML_KEYWORD3); |
147 | } |
148 | state2 = SCE_CAML_DEFAULT, advance = false; |
149 | } |
150 | break; |
151 | |
152 | case SCE_CAML_TAGNAME: |
153 | // [try to] interpret as [additional] tagname char |
154 | if (!(iscaml(sc.ch) || sc.Match('\''))) |
155 | state2 = SCE_CAML_DEFAULT, advance = false; |
156 | break; |
157 | |
158 | /*case SCE_CAML_KEYWORD: |
159 | case SCE_CAML_KEYWORD2: |
160 | case SCE_CAML_KEYWORD3: |
161 | // [try to] interpret as [additional] keyword char |
162 | if (!iscaml(ch)) |
163 | state2 = SCE_CAML_DEFAULT, advance = false; |
164 | break;*/ |
165 | |
166 | case SCE_CAML_LINENUM: |
167 | // [try to] interpret as [additional] linenum directive char |
168 | if (!isdigit(sc.ch)) |
169 | state2 = SCE_CAML_DEFAULT, advance = false; |
170 | break; |
171 | |
172 | case SCE_CAML_OPERATOR: { |
173 | // [try to] interpret as [additional] operator char |
174 | const char* o = 0; |
175 | if (iscaml(sc.ch) || isspace(sc.ch) // ident or whitespace |
176 | || (o = strchr(")]};,\'\"#" , sc.ch),o) // "termination" chars |
177 | || (!isSML && sc.Match('`')) // Caml extra term char |
178 | || (!strchr("!$%&*+-./:<=>?@^|~" , sc.ch)// "operator" chars |
179 | // SML extra ident chars |
180 | && !(isSML && (sc.Match('\\') || sc.Match('`'))))) { |
181 | // check for INCLUSIVE termination |
182 | if (o && strchr(")]};," , sc.ch)) { |
183 | if ((sc.Match(')') && sc.chPrev == '(') |
184 | || (sc.Match(']') && sc.chPrev == '[')) |
185 | // special-case "()" and "[]" tokens as KEYWORDS |
186 | sc.ChangeState(SCE_CAML_KEYWORD); |
187 | chColor++; |
188 | } else |
189 | advance = false; |
190 | state2 = SCE_CAML_DEFAULT; |
191 | } |
192 | break; |
193 | } |
194 | |
195 | case SCE_CAML_NUMBER: |
196 | // [try to] interpret as [additional] numeric literal char |
197 | if ((!isSML && sc.Match('_')) || IsADigit(sc.ch, chBase)) |
198 | break; |
199 | // how about an integer suffix? |
200 | if (!isSML && (sc.Match('l') || sc.Match('L') || sc.Match('n')) |
201 | && (sc.chPrev == '_' || IsADigit(sc.chPrev, chBase))) |
202 | break; |
203 | // or a floating-point literal? |
204 | if (chBase == 10) { |
205 | // with a decimal point? |
206 | if (sc.Match('.') |
207 | && ((!isSML && sc.chPrev == '_') |
208 | || IsADigit(sc.chPrev, chBase))) |
209 | break; |
210 | // with an exponent? (I) |
211 | if ((sc.Match('e') || sc.Match('E')) |
212 | && ((!isSML && (sc.chPrev == '.' || sc.chPrev == '_')) |
213 | || IsADigit(sc.chPrev, chBase))) |
214 | break; |
215 | // with an exponent? (II) |
216 | if (((!isSML && (sc.Match('+') || sc.Match('-'))) |
217 | || (isSML && sc.Match('~'))) |
218 | && (sc.chPrev == 'e' || sc.chPrev == 'E')) |
219 | break; |
220 | } |
221 | // it looks like we have run out of number |
222 | state2 = SCE_CAML_DEFAULT, advance = false; |
223 | break; |
224 | |
225 | case SCE_CAML_CHAR: |
226 | if (!isSML) { |
227 | // [try to] interpret as [additional] char literal char |
228 | if (sc.Match('\\')) { |
229 | chLit = 1; // (definitely IS a char literal) |
230 | if (sc.chPrev == '\\') |
231 | sc.ch = ' '; // (...\\') |
232 | // should we be terminating - one way or another? |
233 | } else if ((sc.Match('\'') && sc.chPrev != '\\') |
234 | || sc.atLineEnd) { |
235 | state2 = SCE_CAML_DEFAULT; |
236 | if (sc.Match('\'')) |
237 | chColor++; |
238 | else |
239 | sc.ChangeState(SCE_CAML_IDENTIFIER); |
240 | // ... maybe a char literal, maybe not |
241 | } else if (chLit < 1 && sc.currentPos - chToken >= 2) |
242 | sc.ChangeState(SCE_CAML_IDENTIFIER), advance = false; |
243 | break; |
244 | }/* else |
245 | // fall through for SML char literal (handle like string) */ |
246 | // Falls through. |
247 | |
248 | case SCE_CAML_STRING: |
249 | // [try to] interpret as [additional] [SML char/] string literal char |
250 | if (isSML && sc.Match('\\') && sc.chPrev != '\\' && isspace(sc.chNext)) |
251 | state2 = SCE_CAML_WHITE; |
252 | else if (sc.Match('\\') && sc.chPrev == '\\') |
253 | sc.ch = ' '; // (...\\") |
254 | // should we be terminating - one way or another? |
255 | else if ((sc.Match('"') && sc.chPrev != '\\') |
256 | || (isSML && sc.atLineEnd)) { |
257 | state2 = SCE_CAML_DEFAULT; |
258 | if (sc.Match('"')) |
259 | chColor++; |
260 | } |
261 | break; |
262 | |
263 | case SCE_CAML_WHITE: |
264 | // [try to] interpret as [additional] SML embedded whitespace char |
265 | if (sc.Match('\\')) { |
266 | // style this puppy NOW... |
267 | state2 = SCE_CAML_STRING, sc.ch = ' ' /* (...\") */, chColor++, |
268 | styler.ColourTo(chColor, SCE_CAML_WHITE), styler.Flush(); |
269 | // ... then backtrack to determine original SML literal type |
270 | Sci_Position p = chColor - 2; |
271 | for (; p >= 0 && styler.StyleAt(p) == SCE_CAML_WHITE; p--) ; |
272 | if (p >= 0) |
273 | state2 = static_cast<int>(styler.StyleAt(p)); |
274 | // take care of state change NOW |
275 | sc.ChangeState(state2), state2 = -1; |
276 | } |
277 | break; |
278 | |
279 | case SCE_CAML_COMMENT: |
280 | case SCE_CAML_COMMENT1: |
281 | case SCE_CAML_COMMENT2: |
282 | case SCE_CAML_COMMENT3: |
283 | // we're IN a comment - does this start a NESTED comment? |
284 | if (sc.Match('(', '*')) |
285 | state2 = sc.state + 1, chToken = sc.currentPos, |
286 | sc.Forward(), sc.ch = ' ' /* (*)... */, nesting++; |
287 | // [try to] interpret as [additional] comment char |
288 | else if (sc.Match(')') && sc.chPrev == '*') { |
289 | if (nesting) |
290 | state2 = (sc.state & 0x0f) - 1, chToken = 0, nesting--; |
291 | else |
292 | state2 = SCE_CAML_DEFAULT; |
293 | chColor++; |
294 | // enable "magic" (read-only) comment AS REQUIRED |
295 | } else if (useMagic && sc.currentPos - chToken == 4 |
296 | && sc.Match('c') && sc.chPrev == 'r' && sc.GetRelative(-2) == '@') |
297 | sc.state |= 0x10; // (switch to read-only comment style) |
298 | break; |
299 | } |
300 | |
301 | // handle state change and char coloring AS REQUIRED |
302 | if (state2 >= 0) |
303 | styler.ColourTo(chColor, sc.state), sc.ChangeState(state2); |
304 | // move to next char UNLESS re-scanning current char |
305 | if (advance) |
306 | sc.Forward(); |
307 | } |
308 | |
309 | // do any required terminal char coloring (JIC) |
310 | sc.Complete(); |
311 | } |
312 | |
313 | static |
314 | void FoldCamlDoc( |
315 | Sci_PositionU, Sci_Position, |
316 | int, |
317 | WordList *[], |
318 | Accessor &) |
319 | { |
320 | } |
321 | |
322 | static const char * const camlWordListDesc[] = { |
323 | "Keywords" , // primary Objective Caml keywords |
324 | "Keywords2" , // "optional" keywords (typically from Pervasives) |
325 | "Keywords3" , // "optional" keywords (typically typenames) |
326 | 0 |
327 | }; |
328 | |
329 | LexerModule lmCaml(SCLEX_CAML, ColouriseCamlDoc, "caml" , FoldCamlDoc, camlWordListDesc); |
330 | |