1/*-------------------------------------------------------------------------
2 *
3 * llvmjit_inline.cpp
4 * Cross module inlining suitable for postgres' JIT
5 *
6 * The inliner iterates over external functions referenced from the passed
7 * module and attempts to inline those. It does so by utilizing pre-built
8 * indexes over both postgres core code and extension modules. When a match
9 * for an external function is found - not guaranteed! - the index will then
10 * be used to judge their instruction count / inline worthiness. After doing
11 * so for all external functions, all the referenced functions (and
12 * prerequisites) will be imported.
13 *
14 * Copyright (c) 2016-2019, PostgreSQL Global Development Group
15 *
16 * IDENTIFICATION
17 * src/backend/lib/llvmjit/llvmjit_inline.cpp
18 *
19 *-------------------------------------------------------------------------
20 */
21
22extern "C"
23{
24#include "postgres.h"
25}
26
27#include "jit/llvmjit.h"
28
29extern "C"
30{
31#include <fcntl.h>
32#include <sys/mman.h>
33#include <sys/stat.h>
34#include <sys/types.h>
35#include <unistd.h>
36
37#include "common/string.h"
38#include "miscadmin.h"
39#include "storage/fd.h"
40}
41
42#include <llvm-c/Core.h>
43#include <llvm-c/BitReader.h>
44
45/* Avoid macro clash with LLVM's C++ headers */
46#undef Min
47
48#include <llvm/ADT/SetVector.h>
49#include <llvm/ADT/StringSet.h>
50#include <llvm/ADT/StringMap.h>
51#include <llvm/Analysis/ModuleSummaryAnalysis.h>
52#if LLVM_VERSION_MAJOR > 3
53#include <llvm/Bitcode/BitcodeReader.h>
54#else
55#include <llvm/Bitcode/ReaderWriter.h>
56#include <llvm/Support/Error.h>
57#endif
58#include <llvm/IR/Attributes.h>
59#include <llvm/IR/CallSite.h>
60#include <llvm/IR/DebugInfo.h>
61#include <llvm/IR/IntrinsicInst.h>
62#include <llvm/IR/IRBuilder.h>
63#include <llvm/IR/ModuleSummaryIndex.h>
64#include <llvm/Linker/IRMover.h>
65#include <llvm/Support/ManagedStatic.h>
66
67
68/*
69 * Type used to represent modules InlineWorkListItem's subject is searched for
70 * in.
71 */
72typedef llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> InlineSearchPath;
73
74/*
75 * Item in queue of to-be-checked symbols and corresponding queue.
76 */
77typedef struct InlineWorkListItem
78{
79 llvm::StringRef symbolName;
80 llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> searchpath;
81} InlineWorkListItem;
82typedef llvm::SmallVector<InlineWorkListItem, 128> InlineWorkList;
83
84/*
85 * Information about symbols processed during inlining. Used to prevent
86 * repeated searches and provide additional information.
87 */
88typedef struct FunctionInlineState
89{
90 int costLimit;
91 bool processed;
92 bool inlined;
93 bool allowReconsidering;
94} FunctionInlineState;
95typedef llvm::StringMap<FunctionInlineState> FunctionInlineStates;
96
97/*
98 * Map of modules that should be inlined, with a list of the to-be inlined
99 * symbols.
100 */
101typedef llvm::StringMap<llvm::StringSet<> > ImportMapTy;
102
103
104const float inline_cost_decay_factor = 0.5;
105const int inline_initial_cost = 150;
106
107/*
108 * These are managed statics so LLVM knows to deallocate them during an
109 * LLVMShutdown(), rather than after (which'd cause crashes).
110 */
111typedef llvm::StringMap<std::unique_ptr<llvm::Module> > ModuleCache;
112llvm::ManagedStatic<ModuleCache> module_cache;
113typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> > SummaryCache;
114llvm::ManagedStatic<SummaryCache> summary_cache;
115
116
117static std::unique_ptr<ImportMapTy> llvm_build_inline_plan(llvm::Module *mod);
118static void llvm_execute_inline_plan(llvm::Module *mod,
119 ImportMapTy *globalsToInline);
120
121static llvm::Module* load_module_cached(llvm::StringRef modPath);
122static std::unique_ptr<llvm::Module> load_module(llvm::StringRef Identifier);
123static std::unique_ptr<llvm::ModuleSummaryIndex> llvm_load_summary(llvm::StringRef path);
124
125
126static llvm::Function* create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
127 llvm::Function *F,
128 llvm::StringRef Name);
129
130static bool function_inlinable(llvm::Function &F,
131 int threshold,
132 FunctionInlineStates &functionState,
133 InlineWorkList &worklist,
134 InlineSearchPath &searchpath,
135 llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
136 int &running_instcount,
137 llvm::StringSet<> &importVars);
138static void function_references(llvm::Function &F,
139 int &running_instcount,
140 llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
141 llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions);
142
143static void add_module_to_inline_search_path(InlineSearchPath& path, llvm::StringRef modpath);
144static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
145summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid);
146
147/* verbose debugging for inliner development */
148/* #define INLINE_DEBUG */
149#ifdef INLINE_DEBUG
150#define ilog elog
151#else
152#define ilog(...) (void) 0
153#endif
154
155/*
156 * Perform inlining of external function references in M based on a simple
157 * cost based analysis.
158 */
159void
160llvm_inline(LLVMModuleRef M)
161{
162 llvm::Module *mod = llvm::unwrap(M);
163
164 std::unique_ptr<ImportMapTy> globalsToInline = llvm_build_inline_plan(mod);
165 if (!globalsToInline)
166 return;
167 llvm_execute_inline_plan(mod, globalsToInline.get());
168}
169
170/*
171 * Build information necessary for inlining external function references in
172 * mod.
173 */
174static std::unique_ptr<ImportMapTy>
175llvm_build_inline_plan(llvm::Module *mod)
176{
177 std::unique_ptr<ImportMapTy> globalsToInline(new ImportMapTy());
178 FunctionInlineStates functionStates;
179 InlineWorkList worklist;
180
181 InlineSearchPath defaultSearchPath;
182
183 /* attempt to add module to search path */
184 add_module_to_inline_search_path(defaultSearchPath, "$libdir/postgres");
185 /* if postgres isn't available, no point continuing */
186 if (defaultSearchPath.empty())
187 return nullptr;
188
189 /*
190 * Start inlining with current references to external functions by putting
191 * them on the inlining worklist. If, during inlining of those, new extern
192 * functions need to be inlined, they'll also be put there, with a lower
193 * priority.
194 */
195 for (const llvm::Function &funcDecl : mod->functions())
196 {
197 InlineWorkListItem item = {};
198 FunctionInlineState inlineState = {};
199
200 /* already has a definition */
201 if (!funcDecl.isDeclaration())
202 continue;
203
204 /* llvm provides implementation */
205 if (funcDecl.isIntrinsic())
206 continue;
207
208 item.symbolName = funcDecl.getName();
209 item.searchpath = defaultSearchPath;
210 worklist.push_back(item);
211 inlineState.costLimit = inline_initial_cost;
212 inlineState.processed = false;
213 inlineState.inlined = false;
214 inlineState.allowReconsidering = false;
215 functionStates[funcDecl.getName()] = inlineState;
216 }
217
218 /*
219 * Iterate over pending worklist items, look them up in index, check
220 * whether they should be inlined.
221 */
222 while (!worklist.empty())
223 {
224 InlineWorkListItem item = worklist.pop_back_val();
225 llvm::StringRef symbolName = item.symbolName;
226 char *cmodname;
227 char *cfuncname;
228 FunctionInlineState &inlineState = functionStates[symbolName];
229 llvm::GlobalValue::GUID funcGUID;
230
231 llvm_split_symbol_name(symbolName.data(), &cmodname, &cfuncname);
232
233 funcGUID = llvm::GlobalValue::getGUID(cfuncname);
234
235 /* already processed */
236 if (inlineState.processed)
237 continue;
238
239
240 if (cmodname)
241 add_module_to_inline_search_path(item.searchpath, cmodname);
242
243 /*
244 * Iterate over all known definitions of function, via the index. Then
245 * look up module(s), check if function actually is defined (there
246 * could be hash conflicts).
247 */
248 for (const auto &gvs : summaries_for_guid(item.searchpath, funcGUID))
249 {
250 const llvm::FunctionSummary *fs;
251 llvm::StringRef modPath = gvs->modulePath();
252 llvm::Module *defMod;
253 llvm::Function *funcDef;
254
255 fs = llvm::cast<llvm::FunctionSummary>(gvs);
256
257#if LLVM_VERSION_MAJOR > 3
258 if (gvs->notEligibleToImport())
259 {
260 ilog(DEBUG1, "ineligibile to import %s due to summary",
261 symbolName.data());
262 continue;
263 }
264#endif
265
266 if ((int) fs->instCount() > inlineState.costLimit)
267 {
268 ilog(DEBUG1, "ineligibile to import %s due to early threshold: %u vs %u",
269 symbolName.data(), fs->instCount(), inlineState.costLimit);
270 inlineState.allowReconsidering = true;
271 continue;
272 }
273
274 defMod = load_module_cached(modPath);
275 if (defMod->materializeMetadata())
276 elog(FATAL, "failed to materialize metadata");
277
278 funcDef = defMod->getFunction(cfuncname);
279
280 /*
281 * This can happen e.g. in case of a hash collision of the
282 * function's name.
283 */
284 if (!funcDef)
285 continue;
286
287 if (funcDef->materialize())
288 elog(FATAL, "failed to materialize metadata");
289
290 Assert(!funcDef->isDeclaration());
291 Assert(funcDef->hasExternalLinkage());
292
293 llvm::StringSet<> importVars;
294 llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions;
295 int running_instcount = 0;
296
297 /*
298 * Check whether function, and objects it depends on, are
299 * inlinable.
300 */
301 if (function_inlinable(*funcDef,
302 inlineState.costLimit,
303 functionStates,
304 worklist,
305 item.searchpath,
306 visitedFunctions,
307 running_instcount,
308 importVars))
309 {
310 /*
311 * Check whether function and all its dependencies are too
312 * big. Dependencies already counted for other functions that
313 * will get inlined are not counted again. While this make
314 * things somewhat order dependent, I can't quite see a point
315 * in a different behaviour.
316 */
317 if (running_instcount > inlineState.costLimit)
318 {
319 ilog(DEBUG1, "skipping inlining of %s due to late threshold %d vs %d",
320 symbolName.data(), running_instcount, inlineState.costLimit);
321 inlineState.allowReconsidering = true;
322 continue;
323 }
324
325 ilog(DEBUG1, "inline top function %s total_instcount: %d, partial: %d",
326 symbolName.data(), running_instcount, fs->instCount());
327
328 /* import referenced function itself */
329 importVars.insert(symbolName);
330
331 {
332 llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath];
333 for (auto& importVar : importVars)
334 modGlobalsToInline.insert(importVar.first());
335 Assert(modGlobalsToInline.size() > 0);
336 }
337
338 /* mark function as inlined */
339 inlineState.inlined = true;
340
341 /*
342 * Found definition to inline, don't look for further
343 * potential definitions.
344 */
345 break;
346 }
347 else
348 {
349 ilog(DEBUG1, "had to skip inlining %s",
350 symbolName.data());
351
352 /* It's possible there's another definition that's inlinable. */
353 }
354 }
355
356 /*
357 * Signal that we're done with symbol, whether successful (inlined =
358 * true above) or not.
359 */
360 inlineState.processed = true;
361 }
362
363 return globalsToInline;
364}
365
366/*
367 * Perform the actual inlining of external functions (and their dependencies)
368 * into mod.
369 */
370static void
371llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)
372{
373 llvm::IRMover Mover(*mod);
374
375 for (const auto& toInline : *globalsToInline)
376 {
377 const llvm::StringRef& modPath = toInline.first();
378 const llvm::StringSet<>& modGlobalsToInline = toInline.second;
379 llvm::SetVector<llvm::GlobalValue *> GlobalsToImport;
380
381 Assert(module_cache->count(modPath));
382 std::unique_ptr<llvm::Module> importMod(std::move((*module_cache)[modPath]));
383 module_cache->erase(modPath);
384
385 if (modGlobalsToInline.empty())
386 continue;
387
388 for (auto &glob: modGlobalsToInline)
389 {
390 llvm::StringRef SymbolName = glob.first();
391 char *modname;
392 char *funcname;
393
394 llvm_split_symbol_name(SymbolName.data(), &modname, &funcname);
395
396 llvm::GlobalValue *valueToImport = importMod->getNamedValue(funcname);
397
398 if (!valueToImport)
399 elog(FATAL, "didn't refind value %s to import", SymbolName.data());
400
401 /*
402 * For functions (global vars are only inlined if already static),
403 * mark imported variables as being clones from other
404 * functions. That a) avoids symbol conflicts b) allows the
405 * optimizer to perform inlining.
406 */
407 if (llvm::isa<llvm::Function>(valueToImport))
408 {
409 llvm::Function *F = llvm::dyn_cast<llvm::Function>(valueToImport);
410 typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
411
412 /*
413 * Per-function info isn't necessarily stripped yet, as the
414 * module is lazy-loaded when stripped above.
415 */
416 llvm::stripDebugInfo(*F);
417
418 /*
419 * If the to-be-imported function is one referenced including
420 * its module name, create a tiny inline function that just
421 * forwards the call. One might think a GlobalAlias would do
422 * the trick, but a) IRMover doesn't override a declaration
423 * with an alias pointing to a definition (instead renaming
424 * it), b) Aliases can't be AvailableExternally.
425 */
426 if (modname)
427 {
428 llvm::Function *AF;
429
430 AF = create_redirection_function(importMod, F, SymbolName);
431
432 GlobalsToImport.insert(AF);
433 llvm::stripDebugInfo(*AF);
434 }
435
436 if (valueToImport->hasExternalLinkage())
437 {
438 valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage);
439 }
440 }
441
442 GlobalsToImport.insert(valueToImport);
443 ilog(DEBUG1, "performing import of %s %s",
444 modPath.data(), SymbolName.data());
445
446 }
447
448#if LLVM_VERSION_MAJOR > 4
449#define IRMOVE_PARAMS , /*IsPerformingImport=*/false
450#elif LLVM_VERSION_MAJOR > 3
451#define IRMOVE_PARAMS , /*LinkModuleInlineAsm=*/false, /*IsPerformingImport=*/false
452#else
453#define IRMOVE_PARAMS
454#endif
455 if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
456 [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {}
457 IRMOVE_PARAMS))
458 elog(FATAL, "function import failed with linker error");
459 }
460}
461
462/*
463 * Return a module identified by modPath, caching it in memory.
464 *
465 * Note that such a module may *not* be modified without copying, otherwise
466 * the cache state would get corrupted.
467 */
468static llvm::Module*
469load_module_cached(llvm::StringRef modPath)
470{
471 auto it = module_cache->find(modPath);
472 if (it == module_cache->end())
473 {
474 it = module_cache->insert(
475 std::make_pair(modPath, load_module(modPath))).first;
476 }
477
478 return it->second.get();
479}
480
481static std::unique_ptr<llvm::Module>
482load_module(llvm::StringRef Identifier)
483{
484 LLVMMemoryBufferRef buf;
485 LLVMModuleRef mod;
486 char path[MAXPGPATH];
487 char *msg;
488
489 snprintf(path, MAXPGPATH,"%s/bitcode/%s", pkglib_path, Identifier.data());
490
491 if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
492 elog(FATAL, "failed to open bitcode file \"%s\": %s",
493 path, msg);
494 if (LLVMGetBitcodeModuleInContext2(LLVMGetGlobalContext(), buf, &mod))
495 elog(FATAL, "failed to parse bitcode in file \"%s\"", path);
496
497 /*
498 * Currently there's no use in more detailed debug info for JITed
499 * code. Until that changes, not much point in wasting memory and cycles
500 * on processing debuginfo.
501 */
502 llvm::StripDebugInfo(*llvm::unwrap(mod));
503
504 return std::unique_ptr<llvm::Module>(llvm::unwrap(mod));
505}
506
507/*
508 * Compute list of referenced variables, functions and the instruction count
509 * for a function.
510 */
511static void
512function_references(llvm::Function &F,
513 int &running_instcount,
514 llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
515 llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions)
516{
517 llvm::SmallPtrSet<const llvm::User *, 32> Visited;
518
519 for (llvm::BasicBlock &BB : F)
520 {
521 for (llvm::Instruction &I : BB)
522 {
523 if (llvm::isa<llvm::DbgInfoIntrinsic>(I))
524 continue;
525
526 llvm::SmallVector<llvm::User *, 8> Worklist;
527 Worklist.push_back(&I);
528
529 running_instcount++;
530
531 while (!Worklist.empty()) {
532 llvm::User *U = Worklist.pop_back_val();
533
534 /* visited before */
535 if (!Visited.insert(U).second)
536 continue;
537
538 for (auto &OI : U->operands()) {
539 llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI);
540 if (!Operand)
541 continue;
542 if (llvm::isa<llvm::BlockAddress>(Operand))
543 continue;
544 if (auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) {
545 referencedVars.insert(GV);
546 if (GV->hasInitializer())
547 Worklist.push_back(GV->getInitializer());
548 continue;
549 }
550 if (auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) {
551 referencedFunctions.insert(CF);
552 continue;
553 }
554 Worklist.push_back(Operand);
555 }
556 }
557 }
558 }
559}
560
561/*
562 * Check whether function F is inlinable and, if so, what globals need to be
563 * imported.
564 *
565 * References to external functions from, potentially recursively, inlined
566 * functions are added to the passed in worklist.
567 */
568static bool
569function_inlinable(llvm::Function &F,
570 int threshold,
571 FunctionInlineStates &functionStates,
572 InlineWorkList &worklist,
573 InlineSearchPath &searchpath,
574 llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
575 int &running_instcount,
576 llvm::StringSet<> &importVars)
577{
578 int subThreshold = threshold * inline_cost_decay_factor;
579 llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars;
580 llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions;
581
582 /* can't rely on what may be inlined */
583 if (F.isInterposable())
584 return false;
585
586 /*
587 * Can't rely on function being present. Alternatively we could create a
588 * static version of these functions?
589 */
590 if (F.hasAvailableExternallyLinkage())
591 return false;
592
593 ilog(DEBUG1, "checking inlinability of %s", F.getName().data());
594
595 if (F.materialize())
596 elog(FATAL, "failed to materialize metadata");
597
598 if (F.getAttributes().hasFnAttribute(llvm::Attribute::NoInline))
599 {
600 ilog(DEBUG1, "ineligibile to import %s due to noinline",
601 F.getName().data());
602 return false;
603 }
604
605 function_references(F, running_instcount, referencedVars, referencedFunctions);
606
607 for (llvm::GlobalVariable* rv: referencedVars)
608 {
609 if (rv->materialize())
610 elog(FATAL, "failed to materialize metadata");
611
612 /*
613 * Never want to inline externally visible vars, cheap enough to
614 * reference.
615 */
616 if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage())
617 continue;
618
619 /*
620 * If variable is file-local, we need to inline it, to be able to
621 * inline the function itself. Can't do that if the variable can be
622 * modified, because they'd obviously get out of sync.
623 *
624 * XXX: Currently not a problem, but there'd be problems with
625 * nontrivial initializers if they were allowed for postgres.
626 */
627 if (!rv->isConstant())
628 {
629 ilog(DEBUG1, "cannot inline %s due to uncloneable variable %s",
630 F.getName().data(), rv->getName().data());
631 return false;
632 }
633
634 ilog(DEBUG1, "memorizing global var %s linkage %d for inlining",
635 rv->getName().data(), (int)rv->getLinkage());
636
637 importVars.insert(rv->getName());
638 /* small cost attributed to each cloned global */
639 running_instcount += 5;
640 }
641
642 visitedFunctions.insert(&F);
643
644 /*
645 * Check referenced functions. Check whether used static ones are
646 * inlinable, and remember external ones for inlining.
647 */
648 for (llvm::Function* referencedFunction: referencedFunctions)
649 {
650 llvm::StringSet<> recImportVars;
651
652 if (referencedFunction->materialize())
653 elog(FATAL, "failed to materialize metadata");
654
655 if (referencedFunction->isIntrinsic())
656 continue;
657
658 /* if already visited skip, otherwise remember */
659 if (!visitedFunctions.insert(referencedFunction).second)
660 continue;
661
662 /*
663 * We don't inline external functions directly here, instead we put
664 * them on the worklist if appropriate and check them from
665 * llvm_build_inline_plan().
666 */
667 if (referencedFunction->hasExternalLinkage())
668 {
669 llvm::StringRef funcName = referencedFunction->getName();
670
671 /*
672 * Don't bother checking for inlining if remaining cost budget is
673 * very small.
674 */
675 if (subThreshold < 5)
676 continue;
677
678 auto it = functionStates.find(funcName);
679 if (it == functionStates.end())
680 {
681 FunctionInlineState inlineState;
682
683 inlineState.costLimit = subThreshold;
684 inlineState.processed = false;
685 inlineState.inlined = false;
686 inlineState.allowReconsidering = false;
687
688 functionStates[funcName] = inlineState;
689 worklist.push_back({funcName, searchpath});
690
691 ilog(DEBUG1,
692 "considering extern function %s at %d for inlining",
693 funcName.data(), subThreshold);
694 }
695 else if (!it->second.inlined &&
696 (!it->second.processed || it->second.allowReconsidering) &&
697 it->second.costLimit < subThreshold)
698 {
699 /*
700 * Update inlining threshold if higher. Need to re-queue
701 * to be processed if already processed with lower
702 * threshold.
703 */
704 if (it->second.processed)
705 {
706 ilog(DEBUG1,
707 "reconsidering extern function %s at %d for inlining, increasing from %d",
708 funcName.data(), subThreshold, it->second.costLimit);
709
710 it->second.processed = false;
711 it->second.allowReconsidering = false;
712 worklist.push_back({funcName, searchpath});
713 }
714 it->second.costLimit = subThreshold;
715 }
716 continue;
717 }
718
719 /* can't rely on what may be inlined */
720 if (referencedFunction->isInterposable())
721 return false;
722
723 if (!function_inlinable(*referencedFunction,
724 subThreshold,
725 functionStates,
726 worklist,
727 searchpath,
728 visitedFunctions,
729 running_instcount,
730 recImportVars))
731 {
732 ilog(DEBUG1,
733 "cannot inline %s due to required function %s not being inlinable",
734 F.getName().data(), referencedFunction->getName().data());
735 return false;
736 }
737
738 /* import referenced function itself */
739 importVars.insert(referencedFunction->getName());
740
741 /* import referenced function and its dependants */
742 for (auto& recImportVar : recImportVars)
743 importVars.insert(recImportVar.first());
744 }
745
746 return true;
747}
748
749/*
750 * Attempt to load module summary located at path. Return empty pointer when
751 * loading fails.
752 */
753static std::unique_ptr<llvm::ModuleSummaryIndex>
754llvm_load_summary(llvm::StringRef path)
755{
756 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr =
757 llvm::MemoryBuffer::getFile(path);
758
759 if (std::error_code EC = MBOrErr.getError())
760 {
761 ilog(DEBUG1, "failed to open %s: %s", path.data(),
762 EC.message().c_str());
763 }
764 else
765 {
766 llvm::MemoryBufferRef ref(*MBOrErr.get().get());
767
768#if LLVM_VERSION_MAJOR > 3
769 llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
770 llvm::getModuleSummaryIndex(ref);
771 if (IndexOrErr)
772 return std::move(IndexOrErr.get());
773 elog(FATAL, "failed to load summary \"%s\": %s",
774 path.data(),
775 toString(IndexOrErr.takeError()).c_str());
776#else
777 llvm::ErrorOr<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
778 llvm::getModuleSummaryIndex(ref, [](const llvm::DiagnosticInfo &) {});
779 if (IndexOrErr)
780 return std::move(IndexOrErr.get());
781 elog(FATAL, "failed to load summary \"%s\": %s",
782 path.data(),
783 IndexOrErr.getError().message().c_str());
784#endif
785 }
786 return nullptr;
787}
788
789/*
790 * Attempt to add modpath to the search path.
791 */
792static void
793add_module_to_inline_search_path(InlineSearchPath& searchpath, llvm::StringRef modpath)
794{
795 /* only extension in libdir are candidates for inlining for now */
796 if (!modpath.startswith("$libdir/"))
797 return;
798
799 /* if there's no match, attempt to load */
800 auto it = summary_cache->find(modpath);
801 if (it == summary_cache->end())
802 {
803 std::string path(modpath);
804 path = path.replace(0, strlen("$libdir"), std::string(pkglib_path) + "/bitcode");
805 path += ".index.bc";
806 (*summary_cache)[modpath] = llvm_load_summary(path);
807 it = summary_cache->find(modpath);
808 }
809
810 Assert(it != summary_cache->end());
811
812 /* if the entry isn't NULL, it's validly loaded */
813 if (it->second)
814 searchpath.push_back(it->second.get());
815}
816
817/*
818 * Search for all references for functions hashing to guid in the search path,
819 * and return them in search path order.
820 */
821static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
822summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid)
823{
824 llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches;
825
826 for (auto index : path)
827 {
828#if LLVM_VERSION_MAJOR > 4
829 llvm::ValueInfo funcVI = index->getValueInfo(guid);
830
831 /* if index doesn't know function, we don't have a body, continue */
832 if (funcVI)
833 for (auto &gv : funcVI.getSummaryList())
834 matches.push_back(gv.get());
835#else
836 const llvm::const_gvsummary_iterator &I =
837 index->findGlobalValueSummaryList(guid);
838 if (I != index->end())
839 {
840 for (auto &gv : I->second)
841 matches.push_back(gv.get());
842 }
843#endif
844 }
845
846 return matches;
847}
848
849/*
850 * Create inline wrapper with the name Name, redirecting the call to F.
851 */
852static llvm::Function*
853create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
854 llvm::Function *F,
855 llvm::StringRef Name)
856{
857 typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
858
859 llvm::LLVMContext &Context = F->getContext();
860 llvm::IRBuilder<> Builder(Context);
861 llvm::Function *AF;
862 llvm::BasicBlock *BB;
863 llvm::CallInst *fwdcall;
864 llvm::Attribute inlineAttribute;
865
866 AF = llvm::Function::Create(F->getFunctionType(),
867 LinkageTypes::AvailableExternallyLinkage,
868 Name, importMod.get());
869 BB = llvm::BasicBlock::Create(Context, "entry", AF);
870
871 Builder.SetInsertPoint(BB);
872 fwdcall = Builder.CreateCall(F, &*AF->arg_begin());
873 inlineAttribute = llvm::Attribute::get(Context,
874 llvm::Attribute::AlwaysInline);
875 fwdcall->addAttribute(~0U, inlineAttribute);
876 Builder.CreateRet(fwdcall);
877
878 return AF;
879}
880