1/*-------------------------------------------------------------------------
2 *
3 * llvmjit.c
4 * Core part of the LLVM JIT provider.
5 *
6 * Copyright (c) 2016-2019, PostgreSQL Global Development Group
7 *
8 * IDENTIFICATION
9 * src/backend/jit/llvm/llvmjit.c
10 *
11 *-------------------------------------------------------------------------
12 */
13
14#include "postgres.h"
15
16#include "jit/llvmjit.h"
17#include "jit/llvmjit_emit.h"
18
19#include "miscadmin.h"
20
21#include "utils/memutils.h"
22#include "utils/resowner_private.h"
23#include "portability/instr_time.h"
24#include "storage/ipc.h"
25
26
27#include <llvm-c/Analysis.h>
28#include <llvm-c/BitReader.h>
29#include <llvm-c/BitWriter.h>
30#include <llvm-c/Core.h>
31#include <llvm-c/ExecutionEngine.h>
32#include <llvm-c/OrcBindings.h>
33#include <llvm-c/Support.h>
34#include <llvm-c/Target.h>
35#include <llvm-c/Transforms/IPO.h>
36#include <llvm-c/Transforms/PassManagerBuilder.h>
37#include <llvm-c/Transforms/Scalar.h>
38#if LLVM_VERSION_MAJOR > 6
39#include <llvm-c/Transforms/Utils.h>
40#endif
41
42
43/* Handle of a module emitted via ORC JIT */
44typedef struct LLVMJitHandle
45{
46 LLVMOrcJITStackRef stack;
47 LLVMOrcModuleHandle orc_handle;
48} LLVMJitHandle;
49
50
51/* types & functions commonly needed for JITing */
52LLVMTypeRef TypeSizeT;
53LLVMTypeRef TypeParamBool;
54LLVMTypeRef TypeStorageBool;
55LLVMTypeRef TypePGFunction;
56LLVMTypeRef StructNullableDatum;
57LLVMTypeRef StructHeapTupleFieldsField3;
58LLVMTypeRef StructHeapTupleFields;
59LLVMTypeRef StructHeapTupleHeaderData;
60LLVMTypeRef StructHeapTupleDataChoice;
61LLVMTypeRef StructHeapTupleData;
62LLVMTypeRef StructMinimalTupleData;
63LLVMTypeRef StructItemPointerData;
64LLVMTypeRef StructBlockId;
65LLVMTypeRef StructFormPgAttribute;
66LLVMTypeRef StructTupleConstr;
67LLVMTypeRef StructTupleDescData;
68LLVMTypeRef StructTupleTableSlot;
69LLVMTypeRef StructHeapTupleTableSlot;
70LLVMTypeRef StructMinimalTupleTableSlot;
71LLVMTypeRef StructMemoryContextData;
72LLVMTypeRef StructPGFinfoRecord;
73LLVMTypeRef StructFmgrInfo;
74LLVMTypeRef StructFunctionCallInfoData;
75LLVMTypeRef StructExprContext;
76LLVMTypeRef StructExprEvalStep;
77LLVMTypeRef StructExprState;
78LLVMTypeRef StructAggState;
79LLVMTypeRef StructAggStatePerGroupData;
80LLVMTypeRef StructAggStatePerTransData;
81
82LLVMValueRef AttributeTemplate;
83LLVMValueRef FuncStrlen;
84LLVMValueRef FuncVarsizeAny;
85LLVMValueRef FuncSlotGetsomeattrsInt;
86LLVMValueRef FuncSlotGetmissingattrs;
87LLVMValueRef FuncMakeExpandedObjectReadOnlyInternal;
88LLVMValueRef FuncExecEvalSubscriptingRef;
89LLVMValueRef FuncExecEvalSysVar;
90LLVMValueRef FuncExecAggTransReparent;
91LLVMValueRef FuncExecAggInitGroup;
92
93
94static bool llvm_session_initialized = false;
95static size_t llvm_generation = 0;
96static const char *llvm_triple = NULL;
97static const char *llvm_layout = NULL;
98
99
100static LLVMTargetMachineRef llvm_opt0_targetmachine;
101static LLVMTargetMachineRef llvm_opt3_targetmachine;
102
103static LLVMTargetRef llvm_targetref;
104static LLVMOrcJITStackRef llvm_opt0_orc;
105static LLVMOrcJITStackRef llvm_opt3_orc;
106
107
108static void llvm_release_context(JitContext *context);
109static void llvm_session_initialize(void);
110static void llvm_shutdown(int code, Datum arg);
111static void llvm_compile_module(LLVMJitContext *context);
112static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
113
114static void llvm_create_types(void);
115static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
116
117
118PG_MODULE_MAGIC;
119
120
121/*
122 * Initialize LLVM JIT provider.
123 */
124void
125_PG_jit_provider_init(JitProviderCallbacks *cb)
126{
127 cb->reset_after_error = llvm_reset_after_error;
128 cb->release_context = llvm_release_context;
129 cb->compile_expr = llvm_compile_expr;
130}
131
132/*
133 * Create a context for JITing work.
134 *
135 * The context, including subsidiary resources, will be cleaned up either when
136 * the context is explicitly released, or when the lifetime of
137 * CurrentResourceOwner ends (usually the end of the current [sub]xact).
138 */
139LLVMJitContext *
140llvm_create_context(int jitFlags)
141{
142 LLVMJitContext *context;
143
144 llvm_assert_in_fatal_section();
145
146 llvm_session_initialize();
147
148 ResourceOwnerEnlargeJIT(CurrentResourceOwner);
149
150 context = MemoryContextAllocZero(TopMemoryContext,
151 sizeof(LLVMJitContext));
152 context->base.flags = jitFlags;
153
154 /* ensure cleanup */
155 context->base.resowner = CurrentResourceOwner;
156 ResourceOwnerRememberJIT(CurrentResourceOwner, PointerGetDatum(context));
157
158 return context;
159}
160
161/*
162 * Release resources required by one llvm context.
163 */
164static void
165llvm_release_context(JitContext *context)
166{
167 LLVMJitContext *llvm_context = (LLVMJitContext *) context;
168
169 llvm_enter_fatal_on_oom();
170
171 /*
172 * When this backend is exiting, don't clean up LLVM. As an error might
173 * have occurred from within LLVM, we do not want to risk reentering. All
174 * resource cleanup is going to happen through process exit.
175 */
176 if (!proc_exit_inprogress)
177 {
178 if (llvm_context->module)
179 {
180 LLVMDisposeModule(llvm_context->module);
181 llvm_context->module = NULL;
182 }
183
184 while (llvm_context->handles != NIL)
185 {
186 LLVMJitHandle *jit_handle;
187
188 jit_handle = (LLVMJitHandle *) linitial(llvm_context->handles);
189 llvm_context->handles = list_delete_first(llvm_context->handles);
190
191 LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
192 pfree(jit_handle);
193 }
194 }
195}
196
197/*
198 * Return module which may be modified, e.g. by creating new functions.
199 */
200LLVMModuleRef
201llvm_mutable_module(LLVMJitContext *context)
202{
203 llvm_assert_in_fatal_section();
204
205 /*
206 * If there's no in-progress module, create a new one.
207 */
208 if (!context->module)
209 {
210 context->compiled = false;
211 context->module_generation = llvm_generation++;
212 context->module = LLVMModuleCreateWithName("pg");
213 LLVMSetTarget(context->module, llvm_triple);
214 LLVMSetDataLayout(context->module, llvm_layout);
215 }
216
217 return context->module;
218}
219
220/*
221 * Expand function name to be non-conflicting. This should be used by code
222 * generating code, when adding new externally visible function definitions to
223 * a Module.
224 */
225char *
226llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
227{
228 Assert(context->module != NULL);
229
230 context->base.instr.created_functions++;
231
232 /*
233 * Previously we used dots to separate, but turns out some tools, e.g.
234 * GDB, don't like that and truncate name.
235 */
236 return psprintf("%s_%zu_%d",
237 basename,
238 context->module_generation,
239 context->counter++);
240}
241
242/*
243 * Return pointer to function funcname, which has to exist. If there's pending
244 * code to be optimized and emitted, do so first.
245 */
246void *
247llvm_get_function(LLVMJitContext *context, const char *funcname)
248{
249 LLVMOrcTargetAddress addr = 0;
250#if defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
251 ListCell *lc;
252#endif
253
254 llvm_assert_in_fatal_section();
255
256 /*
257 * If there is a pending / not emitted module, compile and emit now.
258 * Otherwise we might not find the [correct] function.
259 */
260 if (!context->compiled)
261 {
262 llvm_compile_module(context);
263 }
264
265 /*
266 * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
267 * to mangle here.
268 */
269
270#if defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
271 foreach(lc, context->handles)
272 {
273 LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
274
275 addr = 0;
276 if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
277 elog(ERROR, "failed to look up symbol \"%s\"", funcname);
278 if (addr)
279 return (void *) (uintptr_t) addr;
280 }
281
282#else
283
284#if LLVM_VERSION_MAJOR < 5
285 if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, funcname)))
286 return (void *) (uintptr_t) addr;
287 if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, funcname)))
288 return (void *) (uintptr_t) addr;
289#else
290 if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, funcname))
291 elog(ERROR, "failed to look up symbol \"%s\"", funcname);
292 if (addr)
293 return (void *) (uintptr_t) addr;
294 if (LLVMOrcGetSymbolAddress(llvm_opt3_orc, &addr, funcname))
295 elog(ERROR, "failed to look up symbol \"%s\"", funcname);
296 if (addr)
297 return (void *) (uintptr_t) addr;
298#endif /* LLVM_VERSION_MAJOR */
299
300#endif /* HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN */
301
302 elog(ERROR, "failed to JIT: %s", funcname);
303
304 return NULL;
305}
306
307/*
308 * Return declaration for passed function, adding it to the module if
309 * necessary.
310 *
311 * This is used to make functions imported by llvm_create_types() known to the
312 * module that's currently being worked on.
313 */
314LLVMValueRef
315llvm_get_decl(LLVMModuleRef mod, LLVMValueRef v_src)
316{
317 LLVMValueRef v_fn;
318
319 /* don't repeatedly add function */
320 v_fn = LLVMGetNamedFunction(mod, LLVMGetValueName(v_src));
321 if (v_fn)
322 return v_fn;
323
324 v_fn = LLVMAddFunction(mod,
325 LLVMGetValueName(v_src),
326 LLVMGetElementType(LLVMTypeOf(v_src)));
327 llvm_copy_attributes(v_src, v_fn);
328
329 return v_fn;
330}
331
332/*
333 * Copy attributes from one function to another.
334 */
335void
336llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
337{
338 int num_attributes;
339 int attno;
340 LLVMAttributeRef *attrs;
341
342 num_attributes =
343 LLVMGetAttributeCountAtIndex(v_from, LLVMAttributeFunctionIndex);
344
345 attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
346 LLVMGetAttributesAtIndex(v_from, LLVMAttributeFunctionIndex, attrs);
347
348 for (attno = 0; attno < num_attributes; attno++)
349 {
350 LLVMAddAttributeAtIndex(v_to, LLVMAttributeFunctionIndex,
351 attrs[attno]);
352 }
353}
354
355/*
356 * Return a callable LLVMValueRef for fcinfo.
357 */
358LLVMValueRef
359llvm_function_reference(LLVMJitContext *context,
360 LLVMBuilderRef builder,
361 LLVMModuleRef mod,
362 FunctionCallInfo fcinfo)
363{
364 char *modname;
365 char *basename;
366 char *funcname;
367
368 LLVMValueRef v_fn;
369
370 fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
371
372 if (modname != NULL && basename != NULL)
373 {
374 /* external function in loadable library */
375 funcname = psprintf("pgextern.%s.%s", modname, basename);
376 }
377 else if (basename != NULL)
378 {
379 /* internal function */
380 funcname = psprintf("%s", basename);
381 }
382 else
383 {
384 /*
385 * Function we don't know to handle, return pointer. We do so by
386 * creating a global constant containing a pointer to the function.
387 * Makes IR more readable.
388 */
389 LLVMValueRef v_fn_addr;
390
391 funcname = psprintf("pgoidextern.%u",
392 fcinfo->flinfo->fn_oid);
393 v_fn = LLVMGetNamedGlobal(mod, funcname);
394 if (v_fn != 0)
395 return LLVMBuildLoad(builder, v_fn, "");
396
397 v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
398
399 v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
400 LLVMSetInitializer(v_fn, v_fn_addr);
401 LLVMSetGlobalConstant(v_fn, true);
402
403 return LLVMBuildLoad(builder, v_fn, "");
404 }
405
406 /* check if function already has been added */
407 v_fn = LLVMGetNamedFunction(mod, funcname);
408 if (v_fn != 0)
409 return v_fn;
410
411 v_fn = LLVMAddFunction(mod, funcname, LLVMGetElementType(TypePGFunction));
412
413 return v_fn;
414}
415
416/*
417 * Optimize code in module using the flags set in context.
418 */
419static void
420llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
421{
422 LLVMPassManagerBuilderRef llvm_pmb;
423 LLVMPassManagerRef llvm_mpm;
424 LLVMPassManagerRef llvm_fpm;
425 LLVMValueRef func;
426 int compile_optlevel;
427
428 if (context->base.flags & PGJIT_OPT3)
429 compile_optlevel = 3;
430 else
431 compile_optlevel = 0;
432
433 /*
434 * Have to create a new pass manager builder every pass through, as the
435 * inliner has some per-builder state. Otherwise one ends up only inlining
436 * a function the first time though.
437 */
438 llvm_pmb = LLVMPassManagerBuilderCreate();
439 LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
440 llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
441
442 if (context->base.flags & PGJIT_OPT3)
443 {
444 /* TODO: Unscientifically determined threshold */
445 LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
446 }
447 else
448 {
449 /* we rely on mem2reg heavily, so emit even in the O0 case */
450 LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
451 }
452
453 LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
454
455 /*
456 * Do function level optimization. This could be moved to the point where
457 * functions are emitted, to reduce memory usage a bit.
458 */
459 LLVMInitializeFunctionPassManager(llvm_fpm);
460 for (func = LLVMGetFirstFunction(context->module);
461 func != NULL;
462 func = LLVMGetNextFunction(func))
463 LLVMRunFunctionPassManager(llvm_fpm, func);
464 LLVMFinalizeFunctionPassManager(llvm_fpm);
465 LLVMDisposePassManager(llvm_fpm);
466
467 /*
468 * Perform module level optimization. We do so even in the non-optimized
469 * case, so always-inline functions etc get inlined. It's cheap enough.
470 */
471 llvm_mpm = LLVMCreatePassManager();
472 LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
473 llvm_mpm);
474 /* always use always-inliner pass */
475 if (!(context->base.flags & PGJIT_OPT3))
476 LLVMAddAlwaysInlinerPass(llvm_mpm);
477 /* if doing inlining, but no expensive optimization, add inlining pass */
478 if (context->base.flags & PGJIT_INLINE
479 && !(context->base.flags & PGJIT_OPT3))
480 LLVMAddFunctionInliningPass(llvm_mpm);
481 LLVMRunPassManager(llvm_mpm, context->module);
482 LLVMDisposePassManager(llvm_mpm);
483
484 LLVMPassManagerBuilderDispose(llvm_pmb);
485}
486
487/*
488 * Emit code for the currently pending module.
489 */
490static void
491llvm_compile_module(LLVMJitContext *context)
492{
493 LLVMOrcModuleHandle orc_handle;
494 MemoryContext oldcontext;
495 static LLVMOrcJITStackRef compile_orc;
496 instr_time starttime;
497 instr_time endtime;
498
499 if (context->base.flags & PGJIT_OPT3)
500 compile_orc = llvm_opt3_orc;
501 else
502 compile_orc = llvm_opt0_orc;
503
504 /* perform inlining */
505 if (context->base.flags & PGJIT_INLINE)
506 {
507 INSTR_TIME_SET_CURRENT(starttime);
508 llvm_inline(context->module);
509 INSTR_TIME_SET_CURRENT(endtime);
510 INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
511 endtime, starttime);
512 }
513
514 if (jit_dump_bitcode)
515 {
516 char *filename;
517
518 filename = psprintf("%u.%zu.bc",
519 MyProcPid,
520 context->module_generation);
521 LLVMWriteBitcodeToFile(context->module, filename);
522 pfree(filename);
523 }
524
525
526 /* optimize according to the chosen optimization settings */
527 INSTR_TIME_SET_CURRENT(starttime);
528 llvm_optimize_module(context, context->module);
529 INSTR_TIME_SET_CURRENT(endtime);
530 INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
531 endtime, starttime);
532
533 if (jit_dump_bitcode)
534 {
535 char *filename;
536
537 filename = psprintf("%u.%zu.optimized.bc",
538 MyProcPid,
539 context->module_generation);
540 LLVMWriteBitcodeToFile(context->module, filename);
541 pfree(filename);
542 }
543
544 /*
545 * Emit the code. Note that this can, depending on the optimization
546 * settings, take noticeable resources as code emission executes low-level
547 * instruction combining/selection passes etc. Without optimization a
548 * faster instruction selection mechanism is used.
549 */
550 INSTR_TIME_SET_CURRENT(starttime);
551#if LLVM_VERSION_MAJOR > 6
552 {
553 if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &orc_handle, context->module,
554 llvm_resolve_symbol, NULL))
555 {
556 elog(ERROR, "failed to JIT module");
557 }
558
559 /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
560 }
561#elif LLVM_VERSION_MAJOR > 4
562 {
563 LLVMSharedModuleRef smod;
564
565 smod = LLVMOrcMakeSharedModule(context->module);
566 if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &orc_handle, smod,
567 llvm_resolve_symbol, NULL))
568 {
569 elog(ERROR, "failed to JIT module");
570 }
571 LLVMOrcDisposeSharedModuleRef(smod);
572 }
573#else /* LLVM 4.0 and 3.9 */
574 {
575 orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
576 llvm_resolve_symbol, NULL);
577 LLVMDisposeModule(context->module);
578 }
579#endif
580 INSTR_TIME_SET_CURRENT(endtime);
581 INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
582 endtime, starttime);
583
584 context->module = NULL;
585 context->compiled = true;
586
587 /* remember emitted code for cleanup and lookups */
588 oldcontext = MemoryContextSwitchTo(TopMemoryContext);
589 {
590 LLVMJitHandle *handle;
591
592 handle = (LLVMJitHandle *) palloc(sizeof(LLVMJitHandle));
593 handle->stack = compile_orc;
594 handle->orc_handle = orc_handle;
595
596 context->handles = lappend(context->handles, handle);
597 }
598 MemoryContextSwitchTo(oldcontext);
599
600 ereport(DEBUG1,
601 (errmsg("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
602 INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
603 INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
604 INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
605 errhidestmt(true),
606 errhidecontext(true)));
607}
608
609/*
610 * Per session initialization.
611 */
612static void
613llvm_session_initialize(void)
614{
615 MemoryContext oldcontext;
616 char *error = NULL;
617 char *cpu = NULL;
618 char *features = NULL;
619
620 if (llvm_session_initialized)
621 return;
622
623 oldcontext = MemoryContextSwitchTo(TopMemoryContext);
624
625 LLVMInitializeNativeTarget();
626 LLVMInitializeNativeAsmPrinter();
627 LLVMInitializeNativeAsmParser();
628
629 /*
630 * Synchronize types early, as that also includes inferring the target
631 * triple.
632 */
633 llvm_create_types();
634
635 if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
636 {
637 elog(FATAL, "failed to query triple %s\n", error);
638 }
639
640 /*
641 * We want the generated code to use all available features. Therefore
642 * grab the host CPU string and detect features of the current CPU. The
643 * latter is needed because some CPU architectures default to enabling
644 * features not all CPUs have (weird, huh).
645 */
646 cpu = LLVMGetHostCPUName();
647 features = LLVMGetHostCPUFeatures();
648 elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
649 cpu, features);
650
651 llvm_opt0_targetmachine =
652 LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
653 LLVMCodeGenLevelNone,
654 LLVMRelocDefault,
655 LLVMCodeModelJITDefault);
656 llvm_opt3_targetmachine =
657 LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
658 LLVMCodeGenLevelAggressive,
659 LLVMRelocDefault,
660 LLVMCodeModelJITDefault);
661
662 LLVMDisposeMessage(cpu);
663 cpu = NULL;
664 LLVMDisposeMessage(features);
665 features = NULL;
666
667 /* force symbols in main binary to be loaded */
668 LLVMLoadLibraryPermanently(NULL);
669
670 llvm_opt0_orc = LLVMOrcCreateInstance(llvm_opt0_targetmachine);
671 llvm_opt3_orc = LLVMOrcCreateInstance(llvm_opt3_targetmachine);
672
673#if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
674 if (jit_debugging_support)
675 {
676 LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
677
678 LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
679 LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
680 }
681#endif
682#if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
683 if (jit_profiling_support)
684 {
685 LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
686
687 LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
688 LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
689 }
690#endif
691
692 before_shmem_exit(llvm_shutdown, 0);
693
694 llvm_session_initialized = true;
695
696 MemoryContextSwitchTo(oldcontext);
697}
698
699static void
700llvm_shutdown(int code, Datum arg)
701{
702 /* unregister profiling support, needs to be flushed to be useful */
703
704 if (llvm_opt3_orc)
705 {
706#if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
707 if (jit_profiling_support)
708 LLVMOrcUnregisterPerf(llvm_opt3_orc);
709#endif
710 LLVMOrcDisposeInstance(llvm_opt3_orc);
711 llvm_opt3_orc = NULL;
712 }
713
714 if (llvm_opt0_orc)
715 {
716#if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
717 if (jit_profiling_support)
718 LLVMOrcUnregisterPerf(llvm_opt0_orc);
719#endif
720 LLVMOrcDisposeInstance(llvm_opt0_orc);
721 llvm_opt0_orc = NULL;
722 }
723}
724
725/* helper for llvm_create_types, returning a global var's type */
726static LLVMTypeRef
727load_type(LLVMModuleRef mod, const char *name)
728{
729 LLVMValueRef value;
730 LLVMTypeRef typ;
731
732 /* this'll return a *pointer* to the global */
733 value = LLVMGetNamedGlobal(mod, name);
734 if (!value)
735 elog(ERROR, "type %s is unknown", name);
736
737 /* therefore look at the contained type and return that */
738 typ = LLVMTypeOf(value);
739 Assert(typ != NULL);
740 typ = LLVMGetElementType(typ);
741 Assert(typ != NULL);
742 return typ;
743}
744
745/* helper for llvm_create_types, returning a function's return type */
746static LLVMTypeRef
747load_return_type(LLVMModuleRef mod, const char *name)
748{
749 LLVMValueRef value;
750 LLVMTypeRef typ;
751
752 /* this'll return a *pointer* to the function */
753 value = LLVMGetNamedFunction(mod, name);
754 if (!value)
755 elog(ERROR, "function %s is unknown", name);
756
757 /* get type of function pointer */
758 typ = LLVMTypeOf(value);
759 Assert(typ != NULL);
760 /* dereference pointer */
761 typ = LLVMGetElementType(typ);
762 Assert(typ != NULL);
763 /* and look at return type */
764 typ = LLVMGetReturnType(typ);
765 Assert(typ != NULL);
766
767 return typ;
768}
769
770/*
771 * Load required information, types, function signatures from llvmjit_types.c
772 * and make them available in global variables.
773 *
774 * Those global variables are then used while emitting code.
775 */
776static void
777llvm_create_types(void)
778{
779 char path[MAXPGPATH];
780 LLVMMemoryBufferRef buf;
781 char *msg;
782 LLVMModuleRef mod = NULL;
783
784 snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
785
786 /* open file */
787 if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
788 {
789 elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
790 path, msg);
791 }
792
793 /* eagerly load contents, going to need it all */
794 if (LLVMParseBitcode2(buf, &mod))
795 {
796 elog(ERROR, "LLVMParseBitcode2 of %s failed", path);
797 }
798 LLVMDisposeMemoryBuffer(buf);
799
800 /*
801 * Load triple & layout from clang emitted file so we're guaranteed to be
802 * compatible.
803 */
804 llvm_triple = pstrdup(LLVMGetTarget(mod));
805 llvm_layout = pstrdup(LLVMGetDataLayoutStr(mod));
806
807 TypeSizeT = load_type(mod, "TypeSizeT");
808 TypeParamBool = load_return_type(mod, "FunctionReturningBool");
809 TypeStorageBool = load_type(mod, "TypeStorageBool");
810 TypePGFunction = load_type(mod, "TypePGFunction");
811 StructNullableDatum = load_type(mod, "StructNullableDatum");
812 StructExprContext = load_type(mod, "StructExprContext");
813 StructExprEvalStep = load_type(mod, "StructExprEvalStep");
814 StructExprState = load_type(mod, "StructExprState");
815 StructFunctionCallInfoData = load_type(mod, "StructFunctionCallInfoData");
816 StructMemoryContextData = load_type(mod, "StructMemoryContextData");
817 StructTupleTableSlot = load_type(mod, "StructTupleTableSlot");
818 StructHeapTupleTableSlot = load_type(mod, "StructHeapTupleTableSlot");
819 StructMinimalTupleTableSlot = load_type(mod, "StructMinimalTupleTableSlot");
820 StructHeapTupleData = load_type(mod, "StructHeapTupleData");
821 StructTupleDescData = load_type(mod, "StructTupleDescData");
822 StructAggState = load_type(mod, "StructAggState");
823 StructAggStatePerGroupData = load_type(mod, "StructAggStatePerGroupData");
824 StructAggStatePerTransData = load_type(mod, "StructAggStatePerTransData");
825
826 AttributeTemplate = LLVMGetNamedFunction(mod, "AttributeTemplate");
827 FuncStrlen = LLVMGetNamedFunction(mod, "strlen");
828 FuncVarsizeAny = LLVMGetNamedFunction(mod, "varsize_any");
829 FuncSlotGetsomeattrsInt = LLVMGetNamedFunction(mod, "slot_getsomeattrs_int");
830 FuncSlotGetmissingattrs = LLVMGetNamedFunction(mod, "slot_getmissingattrs");
831 FuncMakeExpandedObjectReadOnlyInternal = LLVMGetNamedFunction(mod, "MakeExpandedObjectReadOnlyInternal");
832 FuncExecEvalSubscriptingRef = LLVMGetNamedFunction(mod, "ExecEvalSubscriptingRef");
833 FuncExecEvalSysVar = LLVMGetNamedFunction(mod, "ExecEvalSysVar");
834 FuncExecAggTransReparent = LLVMGetNamedFunction(mod, "ExecAggTransReparent");
835 FuncExecAggInitGroup = LLVMGetNamedFunction(mod, "ExecAggInitGroup");
836
837 /*
838 * Leave the module alive, otherwise references to function would be
839 * dangling.
840 */
841
842 return;
843}
844
845/*
846 * Split a symbol into module / function parts. If the function is in the
847 * main binary (or an external library) *modname will be NULL.
848 */
849void
850llvm_split_symbol_name(const char *name, char **modname, char **funcname)
851{
852 *modname = NULL;
853 *funcname = NULL;
854
855 /*
856 * Module function names are pgextern.$module.$funcname
857 */
858 if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
859 {
860 /*
861 * Symbol names cannot contain a ., therefore we can split based on
862 * first and last occurrence of one.
863 */
864 *funcname = rindex(name, '.');
865 (*funcname)++; /* jump over . */
866
867 *modname = pnstrdup(name + strlen("pgextern."),
868 *funcname - name - strlen("pgextern.") - 1);
869 Assert(funcname);
870
871 *funcname = pstrdup(*funcname);
872 }
873 else
874 {
875 *modname = NULL;
876 *funcname = pstrdup(name);
877 }
878}
879
880/*
881 * Attempt to resolve symbol, so LLVM can emit a reference to it.
882 */
883static uint64_t
884llvm_resolve_symbol(const char *symname, void *ctx)
885{
886 uintptr_t addr;
887 char *funcname;
888 char *modname;
889
890 /*
891 * macOS prefixes all object level symbols with an underscore. But neither
892 * dlsym() nor PG's inliner expect that. So undo.
893 */
894#if defined(__darwin__)
895 if (symname[0] != '_')
896 elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
897 symname++;
898#endif
899
900 llvm_split_symbol_name(symname, &modname, &funcname);
901
902 /* functions that aren't resolved to names shouldn't ever get here */
903 Assert(funcname);
904
905 if (modname)
906 addr = (uintptr_t) load_external_function(modname, funcname,
907 true, NULL);
908 else
909 addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
910
911 pfree(funcname);
912 if (modname)
913 pfree(modname);
914
915 /* let LLVM will error out - should never happen */
916 if (!addr)
917 elog(WARNING, "failed to resolve name %s", symname);
918
919 return (uint64_t) addr;
920}
921