1/*
2 * This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this
4 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 *
6 * Copyright 1997 - July 2008 CWI, August 2008 - 2019 MonetDB B.V.
7 */
8
9/*
10 * (author) M. Kersten
11 * @+ Session Scenarios
12 * In MonetDB multiple languages, optimizers, and execution engines can
13 * be combined at run time to satisfy a wide user-community.
14 * Such an assemblage of components is called a @emph{scenario}
15 * and consists of a @emph{reader}, @emph{parser}, @emph{optimizer},
16 * @emph{tactic scheduler} and @emph{engine}. These hooks allow
17 * for both linked-in and external components.
18 *
19 * The languages supported are SQL, the Monet Assembly Language (MAL), and profiler.
20 * The default scenario handles MAL instructions, which is used
21 * to illustrate the behavior of the scenario steps.
22 *
23 * The MAL reader component handles interaction with
24 * a front-end to obtain a string for subsequent compilation and
25 * execution. The reader uses the common stream package to read
26 * data in large chunks, if possible. In interactive mode the lines
27 * are processed one at a time.
28 *
29 * The MAL parser component turns the string into
30 * an internal representation of the MAL program.
31 * During this phase semantic checks are performed, such that
32 * we end up with a type correct program.
33 *
34 * The code block is subsequently sent to an MAL optimizer.
35 * In the default case the program is left untouched. For other languages,
36 * the optimizer deploys language specific code transformations,
37 * e.g., foreign-key optimizations in joins and remote query execution.
38 * All optimization information is statically derived from the
39 * code blocks and possible catalogues maintained for the query language
40 * at hand. Optimizers leave advice and their findings in properties
41 * in the symbol table, see @ref{Property Management}.
42 *
43 * Once the program has thus been refined, the
44 * MAL scheduler prepares for execution using tactical optimizations.
45 * For example, it may parallelize the code, generate an ad-hoc
46 * user-defined function, or prepare for efficient replication management.
47 * In the default case, the program is handed over to the MAL interpreter
48 * without any further modification.
49 *
50 * The final stage is to choose an execution paradigm,
51 * i.e. interpretative (default), compilation of an ad-hoc user
52 * defined function, dataflow driven interpretation,
53 * or vectorized pipe-line execution by a dedicated engine.
54 *
55 * A failure encountered in any of the steps terminates the scenario
56 * cycle. It returns to the user for a new command.
57 *
58 * @+ Scenario management
59 * Scenarios are captured in modules; they can be dynamically loaded
60 * and remain active until the system is brought to a halt.
61 * The first time a scenario @sc{xyz} is used, the system looks for a scenario
62 * initialization routine @sc{xyzinitSystem()} and executes it.
63 * It is typically used to prepare the server for language specific interactions.
64 * Thereafter its components are set to those required by
65 * the scenario and the client initialization takes place.
66 *
67 * When the last user interested in a particular scenario leaves the
68 * scene, we activate its finalization routine calling @sc{xyzexitSystem()}.
69 * It typically perform cleanup, backup and monitoring functions.
70 *
71 * A scenario is interpreted in a strictly linear fashion,
72 * i.e. performing a symbolic optimization before scheduling decisions
73 * are taken.
74 * The routines associated with each state in
75 * the scenario may patch the code so as to assure that subsequent
76 * execution can use a different scenario, e.g., to handle dynamic
77 * code fragments.
78 *
79 * The state of execution is maintained in the scenario record for
80 * each individual client. Sharing this information between clients
81 * should be dealt with in the implementation of the scenario managers.
82 * Upon need, the client can postpone a session scenario by
83 * pushing a new one(language, optimize, tactic,
84 * processor). Propagation of the state information is
85 * encapsulated a scenario2scenario() call. Not all transformations
86 * may be legal.
87 *
88 * @+ Scenario administration
89 * Administration of scenarios follows the access rules
90 * defined for code modules in general.
91 *
92 */
93#include "monetdb_config.h"
94#include "mal_scenario.h"
95#include "mal_linker.h" /* for getAddress() */
96#include "mal_client.h"
97#include "mal_authorize.h"
98#include "mal_exception.h"
99#include "mal_profiler.h"
100#include "mal_private.h"
101
102#ifdef HAVE_SYS_TIMES_H
103# include <sys/times.h>
104#endif
105
106static struct SCENARIO scenarioRec[MAXSCEN] = {
107 {"mal", "mal",
108 0, 0, /* hardwired MALinit*/
109 0, 0, /* implicit */
110 "MALinitClient", (MALfcn) &MALinitClient,
111 "MALexitClient", (MALfcn) &MALexitClient,
112 "MALreader", (MALfcn) &MALreader,
113 "MALparser", (MALfcn) &MALparser,
114 "MALoptimizer", 0,
115 0, 0,
116 "MALengine", (MALfcn) &MALengine,
117 "MALcallback", (MALfcn) &MALcallback },
118 {0, 0, /* name */
119 0, 0, /* init */
120 0, 0, /* exit */
121 0, 0, /* initClient */
122 0, 0, /* exitClient */
123 0, 0, /* reader */
124 0, 0, /* parser */
125 0, 0, /* optimizer */
126 0, 0, /* scheduler */
127 0, 0, /* callback */
128 0, 0 /* engine */
129 }
130};
131
132static str fillScenario(Client c, Scenario scen);
133static MT_Lock scenarioLock = MT_LOCK_INITIALIZER("scenarioLock");
134
135
136/*
137 * Currently each user can define a new scenario, provided we have a free slot.
138 * Scenarios not hardwired can always be dropped.
139 */
140Scenario
141getFreeScenario(void)
142{
143 int i;
144 Scenario scen = NULL;
145
146 MT_lock_set(&scenarioLock);
147 for (i = 0; i < MAXSCEN && scenarioRec[i].name; i++)
148 ;
149 if (i < MAXSCEN)
150 scen = scenarioRec + i;
151 MT_lock_unset(&scenarioLock);
152
153 return scen;
154}
155
156/*
157 * A scenario is initialized only once per session.
158 * All other requests are silently ignored.
159 * After initialization, all state functions should have been set.
160 * Initialization includes searching for the scenario startup file in
161 * the etc/MonetDB directory. This creates a dependency, because the
162 * malInclude also needs a scenario. To break this cycle, the system should
163 * call once the routine default scenario for each client first.
164 */
165static str
166initScenario(Client c, Scenario s)
167{
168 str l = s->language;
169 str msg = MAL_SUCCEED;
170
171 if (s->initSystemCmd)
172 return(fillScenario(c, s));
173 /* prepare for conclicts */
174 MT_lock_set(&mal_contextLock);
175 if (s->initSystem && s->initSystemCmd == 0) {
176 s->initSystemCmd = (MALfcn) getAddress(s->initSystem);
177 if (s->initSystemCmd) {
178 msg = (*s->initSystemCmd) (c);
179 } else {
180 char buf[BUFSIZ];
181 snprintf(buf,BUFSIZ,"%s.init", l);
182 msg = createException(MAL, buf, "Scenario not initialized");
183 }
184 }
185 if (msg) {
186 MT_lock_unset(&mal_contextLock);
187 return msg;
188 }
189
190 if (s->exitSystem && s->exitSystemCmd == 0)
191 s->exitSystemCmd = (MALfcn) getAddress(s->exitSystem);
192 if (s->initClient && s->initClientCmd == 0)
193 s->initClientCmd = (MALfcn) getAddress(s->initClient);
194 if (s->exitClient && s->exitClientCmd == 0)
195 s->exitClientCmd = (MALfcn) getAddress(s->exitClient);
196 if (s->reader && s->readerCmd == 0)
197 s->readerCmd = (MALfcn) getAddress(s->reader);
198 if (s->parser && s->parserCmd == 0)
199 s->parserCmd = (MALfcn) getAddress(s->parser);
200 if (s->optimizer && s->optimizerCmd == 0)
201 s->optimizerCmd = (MALfcn) getAddress(s->optimizer);
202 if (s->tactics && s->tacticsCmd == 0)
203 s->tacticsCmd = (MALfcn) getAddress(s->tactics);
204 if (s->callback && s->callbackCmd == 0)
205 s->callbackCmd = (MALfcn) getAddress(s->callback);
206 if (s->engine && s->engineCmd == 0)
207 s->engineCmd = (MALfcn) getAddress(s->engine);
208 MT_lock_unset(&mal_contextLock);
209 return(fillScenario(c, s));
210}
211
212str
213defaultScenario(Client c)
214{
215 return initScenario(c, scenarioRec);
216}
217
218/*
219 * The Monet debugger provides an option to inspect the scenarios currently
220 * defined.
221 *
222 */
223static void
224print_scenarioCommand(stream *f, str cmd, MALfcn funcptr)
225{
226 if (cmd)
227 mnstr_printf(f," \"%s%s\",", cmd, (funcptr?"":"?"));
228 else
229 mnstr_printf(f," nil,");
230}
231
232void
233showScenario(stream *f, Scenario scen)
234{
235 mnstr_printf(f, "[ \"%s\",", scen->name);
236 print_scenarioCommand(f, scen->initSystem, scen->initSystemCmd);
237 print_scenarioCommand(f, scen->exitSystem, scen->exitSystemCmd);
238 print_scenarioCommand(f, scen->initClient, scen->initClientCmd);
239 print_scenarioCommand(f, scen->exitClient, scen->exitClientCmd);
240 print_scenarioCommand(f, scen->parser, scen->parserCmd);
241 print_scenarioCommand(f, scen->optimizer, scen->optimizerCmd);
242 print_scenarioCommand(f, scen->tactics, scen->tacticsCmd);
243 print_scenarioCommand(f, scen->callback, scen->callbackCmd);
244 print_scenarioCommand(f, scen->engine, scen->engineCmd);
245 mnstr_printf(f, "]\n");
246}
247
248Scenario
249findScenario(str nme)
250{
251 int i;
252 Scenario scen = scenarioRec;
253
254 for (i = 0; i < MAXSCEN && scen->name; i++, scen++)
255 if (strcmp(scen->name, nme) == 0)
256 return scen;
257 return NULL;
258}
259
260/*
261 * Functions may become resolved only after the corresponding module
262 * has been loaded. This should be announced as part of the module
263 * prelude code.
264 * Beware that after the update, we also have to adjust the client records.
265 * They contain a copy of the functions addresses.
266 */
267void
268updateScenario(str nme, str fnme, MALfcn fcn)
269{
270 int phase = -1;
271 Scenario scen = findScenario(nme);
272
273 if (scen == NULL)
274 return;
275 if (scen->initSystem && strcmp(scen->initSystem, fnme) == 0)
276 scen->initSystemCmd = fcn;
277 if (scen->exitSystem && strcmp(scen->exitSystem, fnme) == 0)
278 scen->exitSystemCmd = fcn;
279 if (scen->initClient && strcmp(scen->initClient, fnme) == 0) {
280 scen->initClientCmd = fcn;
281 phase = MAL_SCENARIO_INITCLIENT;
282 }
283 if (scen->exitClient && strcmp(scen->exitClient, fnme) == 0) {
284 scen->exitClientCmd = fcn;
285 phase = MAL_SCENARIO_EXITCLIENT;
286 }
287 if (scen->reader && strcmp(scen->reader, fnme) == 0) {
288 scen->readerCmd = fcn;
289 phase = MAL_SCENARIO_READER;
290 }
291 if (scen->parser && strcmp(scen->parser, fnme) == 0) {
292 scen->parserCmd = fcn;
293 phase = MAL_SCENARIO_PARSER;
294 }
295 if (scen->optimizer && strcmp(scen->optimizer, fnme) == 0) {
296 scen->optimizerCmd = fcn;
297 phase = MAL_SCENARIO_OPTIMIZE;
298 }
299 if (scen->tactics && strcmp(scen->tactics, fnme) == 0) {
300 scen->tacticsCmd = fcn;
301 phase = MAL_SCENARIO_SCHEDULER;
302 }
303 if (scen->callback && strcmp(scen->callback, fnme) == 0) {
304 scen->callbackCmd = fcn;
305 phase = MAL_SCENARIO_CALLBACK;
306 }
307 if (scen->engine && strcmp(scen->engine, fnme) == 0) {
308 scen->engineCmd = fcn;
309 phase = MAL_SCENARIO_ENGINE;
310 }
311 if (phase != -1) {
312 Client c1;
313
314 for (c1 = mal_clients; c1 < mal_clients + MAL_MAXCLIENTS; c1++) {
315 if (c1->scenario &&
316 strcmp(c1->scenario, scen->name) == 0)
317 c1->phase[phase] = fcn;
318 if (c1->oldscenario &&
319 strcmp(c1->oldscenario, scen->name) == 0)
320 c1->oldphase[phase] = fcn;
321 }
322 }
323}
324
325void
326showScenarioByName(stream *f, str nme)
327{
328 Scenario scen = findScenario(nme);
329
330 if (scen)
331 showScenario(f, scen);
332}
333
334void
335showAllScenarios(stream *f)
336{
337 int i;
338 Scenario scen = scenarioRec;
339
340 for (i = 0; i < MAXSCEN && scen->name; i++, scen++)
341 showScenario(f, scen);
342}
343
344str getScenarioLanguage(Client c){
345 Scenario scen= findScenario(c->scenario);
346 if( scen) return scen->language;
347 return "mal";
348}
349/*
350 * Changing the scenario for a particular client invalidates the
351 * state maintained for the previous scenario. The old scenario is
352 * retained in the client record to facilitate propagation of
353 * state information, or to simply switch back to the previous one.
354 * Before we initialize a scenario the client scenario is reset to
355 * the MAL scenario. This implies that all scenarios are initialized
356 * using the same scenario. After the scenario initialization file
357 * has been processed, the scenario phases are replaced with the
358 * proper ones.
359 *
360 * All client records should be initialized with a default
361 * scenario, i.e. the first described in the scenario table.
362 */
363static str
364fillScenario(Client c, Scenario scen)
365{
366 c->scenario = scen->name;
367
368 c->phase[MAL_SCENARIO_READER] = scen->readerCmd;
369 c->phase[MAL_SCENARIO_PARSER] = scen->parserCmd;
370 c->phase[MAL_SCENARIO_OPTIMIZE] = scen->optimizerCmd;
371 c->phase[MAL_SCENARIO_SCHEDULER] = scen->tacticsCmd;
372 c->phase[MAL_SCENARIO_CALLBACK] = scen->callbackCmd;
373 c->phase[MAL_SCENARIO_ENGINE] = scen->engineCmd;
374 c->phase[MAL_SCENARIO_INITCLIENT] = scen->initClientCmd;
375 c->phase[MAL_SCENARIO_EXITCLIENT] = scen->exitClientCmd;
376 c->state[MAL_SCENARIO_READER] = 0;
377 c->state[MAL_SCENARIO_PARSER] = 0;
378 c->state[MAL_SCENARIO_OPTIMIZE] = 0;
379 c->state[MAL_SCENARIO_SCHEDULER] = 0;
380 c->state[MAL_SCENARIO_ENGINE] = 0;
381 c->state[MAL_SCENARIO_INITCLIENT] = 0;
382 c->state[MAL_SCENARIO_EXITCLIENT] = 0;
383 return(MAL_SUCCEED);
384}
385
386/*
387 * Setting a new scenario calls for saving the previous state
388 * and execution of the initClientScenario routine.
389 */
390str
391setScenario(Client c, str nme)
392{
393 int i;
394 str msg;
395 Scenario scen;
396
397 scen = findScenario(nme);
398 if (scen == NULL)
399 throw(MAL, "setScenario", SCENARIO_NOT_FOUND " '%s'", nme);
400
401 if (c->scenario) {
402 c->oldscenario = c->scenario;
403 for (i = 0; i < SCENARIO_PROPERTIES; i++) {
404 c->oldstate[i] = c->state[i];
405 c->oldphase[i] = c->phase[i];
406 }
407 }
408 for (i = 0; i < SCENARIO_PROPERTIES; i++)
409 c->state[i] = 0;
410
411 msg = initScenario(c, scen);
412 if (msg) {
413 /* error occurred, reset the scenario , assume default always works */
414 c->scenario = c->oldscenario;
415 for (i = 0; i < SCENARIO_PROPERTIES; i++) {
416 c->state[i] = c->oldstate[i];
417 c->phase[i] = c->oldphase[i];
418 c->oldstate[i] = NULL;
419 c->oldphase[i] = NULL;
420 }
421 c->oldscenario = NULL;
422 return msg;
423 }
424 return MAL_SUCCEED;
425}
426
427/*
428 * After finishing a session in a scenario, we should reset the
429 * state of the previous one. But also call the exitClient
430 * to garbage collect any scenario specific structures.
431 */
432#if 0
433str
434getCurrentScenario(Client c)
435{
436 return c->scenario;
437}
438#endif
439
440void
441resetScenario(Client c)
442{
443 int i;
444 Scenario scen = scenarioRec;
445
446 if (c->scenario == 0)
447 return;
448
449 scen = findScenario(c->scenario);
450 if (scen != NULL && scen->exitClientCmd)
451 (*scen->exitClientCmd) (c);
452
453 c->scenario = c->oldscenario;
454 for (i = 0; i < SCENARIO_PROPERTIES; i++) {
455 c->state[i] = c->oldstate[i];
456 c->phase[i] = c->oldphase[i];
457 }
458 c->oldscenario = 0;
459}
460
461/*
462 * The building blocks of scenarios are routines obeying a strict
463 * name signature. They require exclusive access to the client
464 * record. Any specific information should be accessible from
465 * there, e.g., access to a scenario specific state descriptor.
466 * The client scenario initialization and finalization brackets
467 * are @sc{xyzinitClient()} and @sc{xyzexitClient()}.
468 *
469 * The @sc{xyzparser(Client c)} contains the parser for language XYZ
470 * and should fill the MAL program block associated with the client record.
471 * The latter may have been initialized with variables.
472 * Each language parser may require a catalog with information
473 * on the translation of language specific datastructures into their BAT
474 * equivalent.
475 *
476 * The @sc{xyzoptimizer(Client c)} contains language specific optimizations
477 * using the MAL intermediate code as a starting point.
478 *
479 * The @sc{xyztactics(Client c)} synchronizes the program execution with the
480 * state of the machine, e.g., claiming resources, the history of the client
481 * or alignment of the request with concurrent actions (e.g., transaction
482 * coordination).
483 *
484 * The @sc{xyzengine(Client c)} contains the applicable back-end engine.
485 * The default is the MAL interpreter, which provides good balance
486 * between speed and ability to analysis its behavior.
487 *
488 */
489static const char *phases[] = {
490 [MAL_SCENARIO_CALLBACK] = "scenario callback",
491 [MAL_SCENARIO_ENGINE] = "scenario engine",
492 [MAL_SCENARIO_EXITCLIENT] = "scenario exitclient",
493 [MAL_SCENARIO_INITCLIENT] = "scenario initclient",
494 [MAL_SCENARIO_OPTIMIZE] = "scenario optimize",
495 [MAL_SCENARIO_PARSER] = "scenario parser",
496 [MAL_SCENARIO_READER] = "scenario reader",
497 [MAL_SCENARIO_SCHEDULER] = "scenario scheduler",
498};
499static str
500runPhase(Client c, int phase)
501{
502 str msg = MAL_SUCCEED;
503 if (c->phase[phase]) {
504 MT_thread_setworking(phases[phase]);
505 return msg = (str) (*c->phase[phase])(c);
506 }
507 return msg;
508}
509
510/*
511 * Access control enforcement. Except for the server owner
512 * running a scenario should be explicitly permitted.
513 */
514static str
515runScenarioBody(Client c, int once)
516{
517 str msg = MAL_SUCCEED;
518
519 while (c->mode > FINISHCLIENT && !GDKexiting()) {
520 // be aware that a MAL call may initialize a different scenario
521 if ( !c->state[0] && (msg = runPhase(c, MAL_SCENARIO_INITCLIENT)) )
522 goto wrapup;
523 if ( c->mode <= FINISHCLIENT || (msg = runPhase(c, MAL_SCENARIO_READER)) )
524 goto wrapup;
525 if ( c->mode <= FINISHCLIENT || (msg = runPhase(c, MAL_SCENARIO_PARSER)) || c->blkmode)
526 goto wrapup;
527 if ( c->mode <= FINISHCLIENT || (msg = runPhase(c, MAL_SCENARIO_OPTIMIZE)) )
528 goto wrapup;
529 if ( c->mode <= FINISHCLIENT || (msg = runPhase(c, MAL_SCENARIO_SCHEDULER)))
530 goto wrapup;
531 if ( c->mode <= FINISHCLIENT || (msg = runPhase(c, MAL_SCENARIO_ENGINE)))
532 goto wrapup;
533 wrapup:
534 if (msg != MAL_SUCCEED){
535 if (c->phase[MAL_SCENARIO_CALLBACK]) {
536 MT_thread_setworking(phases[MAL_SCENARIO_CALLBACK]);
537 msg = (str) (*c->phase[MAL_SCENARIO_CALLBACK])(c, msg);
538 }
539 if (msg) {
540 mnstr_printf(c->fdout,"!%s%s", msg, (msg[strlen(msg)-1] == '\n'? "":"\n"));
541 freeException(msg);
542 msg = MAL_SUCCEED;
543 }
544 }
545 if( GDKerrbuf && GDKerrbuf[0])
546 mnstr_printf(c->fdout,"!GDKerror: %s\n",GDKerrbuf);
547 assert(c->curprg->def->errors == NULL);
548 c->actions++;
549 if( once) break;
550 }
551 if (once == 0)
552 msg = runPhase(c, MAL_SCENARIO_EXITCLIENT);
553 return msg;
554}
555
556str
557runScenario(Client c, int once)
558{
559 str msg = MAL_SUCCEED;
560
561 if (c == 0 || c->phase[MAL_SCENARIO_READER] == 0)
562 return msg;
563 msg = runScenarioBody(c,once);
564 if (msg != MAL_SUCCEED &&
565 strcmp(msg,"MALException:client.quit:Server stopped."))
566 mnstr_printf(c->fdout,"!%s\n",msg);
567 return msg;
568}
569
570