1// Scintilla source code edit control
2/** @file LexCaml.cxx
3 ** Lexer for Objective Caml.
4 **/
5// Copyright 2005-2009 by Robert Roessler <robertr@rftp.com>
6// The License.txt file describes the conditions under which this software may be distributed.
7/* Release History
8 20050204 Initial release.
9 20050205 Quick compiler standards/"cleanliness" adjustment.
10 20050206 Added cast for IsLeadByte().
11 20050209 Changes to "external" build support.
12 20050306 Fix for 1st-char-in-doc "corner" case.
13 20050502 Fix for [harmless] one-past-the-end coloring.
14 20050515 Refined numeric token recognition logic.
15 20051125 Added 2nd "optional" keywords class.
16 20051129 Support "magic" (read-only) comments for RCaml.
17 20051204 Swtich to using StyleContext infrastructure.
18 20090629 Add full Standard ML '97 support.
19*/
20
21#include <stdlib.h>
22#include <string.h>
23#include <stdio.h>
24#include <stdarg.h>
25#include <assert.h>
26#include <ctype.h>
27
28#include <string>
29#include <string_view>
30
31#include "ILexer.h"
32#include "Scintilla.h"
33#include "SciLexer.h"
34
35#include "WordList.h"
36#include "LexAccessor.h"
37#include "Accessor.h"
38#include "StyleContext.h"
39#include "CharacterSet.h"
40#include "LexerModule.h"
41
42#if defined(__clang__)
43#pragma clang diagnostic ignored "-Wcomma"
44#endif
45
46// Since the Microsoft __iscsym[f] funcs are not ANSI...
47inline int iscaml(int c) {return isalnum(c) || c == '_';}
48inline int iscamlf(int c) {return isalpha(c) || c == '_';}
49
50static const int baseT[24] = {
51 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* A - L */
52 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0,16 /* M - X */
53};
54
55using namespace Lexilla;
56
57static void ColouriseCamlDoc(
58 Sci_PositionU startPos, Sci_Position length,
59 int initStyle,
60 WordList *keywordlists[],
61 Accessor &styler)
62{
63 // initialize styler
64 StyleContext sc(startPos, length, initStyle, styler);
65
66 Sci_PositionU chToken = 0;
67 int chBase = 0, chLit = 0;
68 WordList& keywords = *keywordlists[0];
69 WordList& keywords2 = *keywordlists[1];
70 WordList& keywords3 = *keywordlists[2];
71 const bool isSML = keywords.InList("andalso");
72 const int useMagic = styler.GetPropertyInt("lexer.caml.magic", 0);
73
74 // set up [initial] state info (terminating states that shouldn't "bleed")
75 const int state_ = sc.state & 0x0f;
76 if (state_ <= SCE_CAML_CHAR
77 || (isSML && state_ == SCE_CAML_STRING))
78 sc.state = SCE_CAML_DEFAULT;
79 int nesting = (state_ >= SCE_CAML_COMMENT)? (state_ - SCE_CAML_COMMENT): 0;
80
81 // foreach char in range...
82 while (sc.More()) {
83 // set up [per-char] state info
84 int state2 = -1; // (ASSUME no state change)
85 Sci_Position chColor = sc.currentPos - 1;// (ASSUME standard coloring range)
86 bool advance = true; // (ASSUME scanner "eats" 1 char)
87
88 // step state machine
89 switch (sc.state & 0x0f) {
90 case SCE_CAML_DEFAULT:
91 chToken = sc.currentPos; // save [possible] token start (JIC)
92 // it's wide open; what do we have?
93 if (iscamlf(sc.ch))
94 state2 = SCE_CAML_IDENTIFIER;
95 else if (!isSML && sc.Match('`') && iscamlf(sc.chNext))
96 state2 = SCE_CAML_TAGNAME;
97 else if (!isSML && sc.Match('#') && isdigit(sc.chNext))
98 state2 = SCE_CAML_LINENUM;
99 else if (isdigit(sc.ch)) {
100 // it's a number, assume base 10
101 state2 = SCE_CAML_NUMBER, chBase = 10;
102 if (sc.Match('0')) {
103 // there MAY be a base specified...
104 const char* baseC = "bBoOxX";
105 if (isSML) {
106 if (sc.chNext == 'w')
107 sc.Forward(); // (consume SML "word" indicator)
108 baseC = "x";
109 }
110 // ... change to specified base AS REQUIRED
111 if (strchr(baseC, sc.chNext))
112 chBase = baseT[tolower(sc.chNext) - 'a'], sc.Forward();
113 }
114 } else if (!isSML && sc.Match('\'')) // (Caml char literal?)
115 state2 = SCE_CAML_CHAR, chLit = 0;
116 else if (isSML && sc.Match('#', '"')) // (SML char literal?)
117 state2 = SCE_CAML_CHAR, sc.Forward();
118 else if (sc.Match('"'))
119 state2 = SCE_CAML_STRING;
120 else if (sc.Match('(', '*'))
121 state2 = SCE_CAML_COMMENT, sc.Forward(), sc.ch = ' '; // (*)...
122 else if (strchr("!?~" /* Caml "prefix-symbol" */
123 "=<>@^|&+-*/$%" /* Caml "infix-symbol" */
124 "()[]{};,:.#", sc.ch) // Caml "bracket" or ;,:.#
125 // SML "extra" ident chars
126 || (isSML && (sc.Match('\\') || sc.Match('`'))))
127 state2 = SCE_CAML_OPERATOR;
128 break;
129
130 case SCE_CAML_IDENTIFIER:
131 // [try to] interpret as [additional] identifier char
132 if (!(iscaml(sc.ch) || sc.Match('\''))) {
133 const Sci_Position n = sc.currentPos - chToken;
134 if (n < 24) {
135 // length is believable as keyword, [re-]construct token
136 char t[24];
137 for (Sci_Position i = -n; i < 0; i++)
138 t[n + i] = static_cast<char>(sc.GetRelative(i));
139 t[n] = '\0';
140 // special-case "_" token as KEYWORD
141 if ((n == 1 && sc.chPrev == '_') || keywords.InList(t))
142 sc.ChangeState(SCE_CAML_KEYWORD);
143 else if (keywords2.InList(t))
144 sc.ChangeState(SCE_CAML_KEYWORD2);
145 else if (keywords3.InList(t))
146 sc.ChangeState(SCE_CAML_KEYWORD3);
147 }
148 state2 = SCE_CAML_DEFAULT, advance = false;
149 }
150 break;
151
152 case SCE_CAML_TAGNAME:
153 // [try to] interpret as [additional] tagname char
154 if (!(iscaml(sc.ch) || sc.Match('\'')))
155 state2 = SCE_CAML_DEFAULT, advance = false;
156 break;
157
158 /*case SCE_CAML_KEYWORD:
159 case SCE_CAML_KEYWORD2:
160 case SCE_CAML_KEYWORD3:
161 // [try to] interpret as [additional] keyword char
162 if (!iscaml(ch))
163 state2 = SCE_CAML_DEFAULT, advance = false;
164 break;*/
165
166 case SCE_CAML_LINENUM:
167 // [try to] interpret as [additional] linenum directive char
168 if (!isdigit(sc.ch))
169 state2 = SCE_CAML_DEFAULT, advance = false;
170 break;
171
172 case SCE_CAML_OPERATOR: {
173 // [try to] interpret as [additional] operator char
174 const char* o = 0;
175 if (iscaml(sc.ch) || isspace(sc.ch) // ident or whitespace
176 || (o = strchr(")]};,\'\"#", sc.ch),o) // "termination" chars
177 || (!isSML && sc.Match('`')) // Caml extra term char
178 || (!strchr("!$%&*+-./:<=>?@^|~", sc.ch)// "operator" chars
179 // SML extra ident chars
180 && !(isSML && (sc.Match('\\') || sc.Match('`'))))) {
181 // check for INCLUSIVE termination
182 if (o && strchr(")]};,", sc.ch)) {
183 if ((sc.Match(')') && sc.chPrev == '(')
184 || (sc.Match(']') && sc.chPrev == '['))
185 // special-case "()" and "[]" tokens as KEYWORDS
186 sc.ChangeState(SCE_CAML_KEYWORD);
187 chColor++;
188 } else
189 advance = false;
190 state2 = SCE_CAML_DEFAULT;
191 }
192 break;
193 }
194
195 case SCE_CAML_NUMBER:
196 // [try to] interpret as [additional] numeric literal char
197 if ((!isSML && sc.Match('_')) || IsADigit(sc.ch, chBase))
198 break;
199 // how about an integer suffix?
200 if (!isSML && (sc.Match('l') || sc.Match('L') || sc.Match('n'))
201 && (sc.chPrev == '_' || IsADigit(sc.chPrev, chBase)))
202 break;
203 // or a floating-point literal?
204 if (chBase == 10) {
205 // with a decimal point?
206 if (sc.Match('.')
207 && ((!isSML && sc.chPrev == '_')
208 || IsADigit(sc.chPrev, chBase)))
209 break;
210 // with an exponent? (I)
211 if ((sc.Match('e') || sc.Match('E'))
212 && ((!isSML && (sc.chPrev == '.' || sc.chPrev == '_'))
213 || IsADigit(sc.chPrev, chBase)))
214 break;
215 // with an exponent? (II)
216 if (((!isSML && (sc.Match('+') || sc.Match('-')))
217 || (isSML && sc.Match('~')))
218 && (sc.chPrev == 'e' || sc.chPrev == 'E'))
219 break;
220 }
221 // it looks like we have run out of number
222 state2 = SCE_CAML_DEFAULT, advance = false;
223 break;
224
225 case SCE_CAML_CHAR:
226 if (!isSML) {
227 // [try to] interpret as [additional] char literal char
228 if (sc.Match('\\')) {
229 chLit = 1; // (definitely IS a char literal)
230 if (sc.chPrev == '\\')
231 sc.ch = ' '; // (...\\')
232 // should we be terminating - one way or another?
233 } else if ((sc.Match('\'') && sc.chPrev != '\\')
234 || sc.atLineEnd) {
235 state2 = SCE_CAML_DEFAULT;
236 if (sc.Match('\''))
237 chColor++;
238 else
239 sc.ChangeState(SCE_CAML_IDENTIFIER);
240 // ... maybe a char literal, maybe not
241 } else if (chLit < 1 && sc.currentPos - chToken >= 2)
242 sc.ChangeState(SCE_CAML_IDENTIFIER), advance = false;
243 break;
244 }/* else
245 // fall through for SML char literal (handle like string) */
246 // Falls through.
247
248 case SCE_CAML_STRING:
249 // [try to] interpret as [additional] [SML char/] string literal char
250 if (isSML && sc.Match('\\') && sc.chPrev != '\\' && isspace(sc.chNext))
251 state2 = SCE_CAML_WHITE;
252 else if (sc.Match('\\') && sc.chPrev == '\\')
253 sc.ch = ' '; // (...\\")
254 // should we be terminating - one way or another?
255 else if ((sc.Match('"') && sc.chPrev != '\\')
256 || (isSML && sc.atLineEnd)) {
257 state2 = SCE_CAML_DEFAULT;
258 if (sc.Match('"'))
259 chColor++;
260 }
261 break;
262
263 case SCE_CAML_WHITE:
264 // [try to] interpret as [additional] SML embedded whitespace char
265 if (sc.Match('\\')) {
266 // style this puppy NOW...
267 state2 = SCE_CAML_STRING, sc.ch = ' ' /* (...\") */, chColor++,
268 styler.ColourTo(chColor, SCE_CAML_WHITE), styler.Flush();
269 // ... then backtrack to determine original SML literal type
270 Sci_Position p = chColor - 2;
271 for (; p >= 0 && styler.StyleAt(p) == SCE_CAML_WHITE; p--) ;
272 if (p >= 0)
273 state2 = static_cast<int>(styler.StyleAt(p));
274 // take care of state change NOW
275 sc.ChangeState(state2), state2 = -1;
276 }
277 break;
278
279 case SCE_CAML_COMMENT:
280 case SCE_CAML_COMMENT1:
281 case SCE_CAML_COMMENT2:
282 case SCE_CAML_COMMENT3:
283 // we're IN a comment - does this start a NESTED comment?
284 if (sc.Match('(', '*'))
285 state2 = sc.state + 1, chToken = sc.currentPos,
286 sc.Forward(), sc.ch = ' ' /* (*)... */, nesting++;
287 // [try to] interpret as [additional] comment char
288 else if (sc.Match(')') && sc.chPrev == '*') {
289 if (nesting)
290 state2 = (sc.state & 0x0f) - 1, chToken = 0, nesting--;
291 else
292 state2 = SCE_CAML_DEFAULT;
293 chColor++;
294 // enable "magic" (read-only) comment AS REQUIRED
295 } else if (useMagic && sc.currentPos - chToken == 4
296 && sc.Match('c') && sc.chPrev == 'r' && sc.GetRelative(-2) == '@')
297 sc.state |= 0x10; // (switch to read-only comment style)
298 break;
299 }
300
301 // handle state change and char coloring AS REQUIRED
302 if (state2 >= 0)
303 styler.ColourTo(chColor, sc.state), sc.ChangeState(state2);
304 // move to next char UNLESS re-scanning current char
305 if (advance)
306 sc.Forward();
307 }
308
309 // do any required terminal char coloring (JIC)
310 sc.Complete();
311}
312
313static
314void FoldCamlDoc(
315 Sci_PositionU, Sci_Position,
316 int,
317 WordList *[],
318 Accessor &)
319{
320}
321
322static const char * const camlWordListDesc[] = {
323 "Keywords", // primary Objective Caml keywords
324 "Keywords2", // "optional" keywords (typically from Pervasives)
325 "Keywords3", // "optional" keywords (typically typenames)
326 0
327};
328
329LexerModule lmCaml(SCLEX_CAML, ColouriseCamlDoc, "caml", FoldCamlDoc, camlWordListDesc);
330