fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #72883 - Mark-Simulacrum:stable-next, r=Mark-Simulacrum49cae55760d

1.44.0
51.0 KiB1547 linesraw
1#include <stdio.h>
2
3#include <vector>
4#include <set>
5
6#include "rustllvm.h"
7
8#include "llvm/Analysis/TargetLibraryInfo.h"
9#include "llvm/Analysis/TargetTransformInfo.h"
10#include "llvm/CodeGen/TargetSubtargetInfo.h"
11#include "llvm/InitializePasses.h"
12#include "llvm/IR/AutoUpgrade.h"
13#include "llvm/IR/AssemblyAnnotationWriter.h"
14#include "llvm/IR/IntrinsicInst.h"
15#include "llvm/IR/Verifier.h"
16#include "llvm/Passes/PassBuilder.h"
17#if LLVM_VERSION_GE(9, 0)
18#include "llvm/Passes/StandardInstrumentations.h"
19#endif
20#include "llvm/Support/CBindingWrapping.h"
21#include "llvm/Support/FileSystem.h"
22#include "llvm/Support/Host.h"
23#include "llvm/Target/TargetMachine.h"
24#include "llvm/Transforms/IPO/PassManagerBuilder.h"
25#include "llvm/Transforms/IPO/AlwaysInliner.h"
26#include "llvm/Transforms/IPO/FunctionImport.h"
27#include "llvm/Transforms/Utils/FunctionImportUtils.h"
28#include "llvm/LTO/LTO.h"
29#include "llvm-c/Transforms/PassManagerBuilder.h"
30
31#include "llvm/Transforms/Instrumentation.h"
32#if LLVM_VERSION_GE(9, 0)
33#include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
34#include "llvm/Support/TimeProfiler.h"
35#endif
36#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
37#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
38#if LLVM_VERSION_GE(9, 0)
39#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
40#endif
41#include "llvm/Transforms/Utils/NameAnonGlobals.h"
42
43using namespace llvm;
44
45typedef struct LLVMOpaquePass *LLVMPassRef;
46typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
47
48DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
49DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
50DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder,
51                                   LLVMPassManagerBuilderRef)
52
53extern "C" void LLVMInitializePasses() {
54  PassRegistry &Registry = *PassRegistry::getPassRegistry();
55  initializeCore(Registry);
56  initializeCodeGen(Registry);
57  initializeScalarOpts(Registry);
58  initializeVectorization(Registry);
59  initializeIPO(Registry);
60  initializeAnalysis(Registry);
61  initializeTransformUtils(Registry);
62  initializeInstCombine(Registry);
63  initializeInstrumentation(Registry);
64  initializeTarget(Registry);
65}
66
67extern "C" void LLVMTimeTraceProfilerInitialize() {
68#if LLVM_VERSION_GE(10, 0)
69  timeTraceProfilerInitialize(
70      /* TimeTraceGranularity */ 0,
71      /* ProcName */ "rustc");
72#elif LLVM_VERSION_GE(9, 0)
73  timeTraceProfilerInitialize();
74#endif
75}
76
77extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
78#if LLVM_VERSION_GE(9, 0)
79  StringRef FN(FileName);
80  std::error_code EC;
81  raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
82
83  timeTraceProfilerWrite(OS);
84  timeTraceProfilerCleanup();
85#endif
86}
87
88enum class LLVMRustPassKind {
89  Other,
90  Function,
91  Module,
92};
93
94static LLVMRustPassKind toRust(PassKind Kind) {
95  switch (Kind) {
96  case PT_Function:
97    return LLVMRustPassKind::Function;
98  case PT_Module:
99    return LLVMRustPassKind::Module;
100  default:
101    return LLVMRustPassKind::Other;
102  }
103}
104
105extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
106  StringRef SR(PassName);
107  PassRegistry *PR = PassRegistry::getPassRegistry();
108
109  const PassInfo *PI = PR->getPassInfo(SR);
110  if (PI) {
111    return wrap(PI->createPass());
112  }
113  return nullptr;
114}
115
116extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
117  const bool CompileKernel = false;
118  const bool UseAfterScope = true;
119
120  return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope));
121}
122
123extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
124  const bool CompileKernel = false;
125
126#if LLVM_VERSION_GE(9, 0)
127  return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover));
128#else
129  return wrap(createAddressSanitizerModulePass(CompileKernel, Recover));
130#endif
131}
132
133extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) {
134#if LLVM_VERSION_GE(9, 0)
135  const bool CompileKernel = false;
136
137  return wrap(createMemorySanitizerLegacyPassPass(
138      MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel}));
139#else
140  return wrap(createMemorySanitizerLegacyPassPass(TrackOrigins, Recover));
141#endif
142}
143
144extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() {
145  return wrap(createThreadSanitizerLegacyPassPass());
146}
147
148extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) {
149  assert(RustPass);
150  Pass *Pass = unwrap(RustPass);
151  return toRust(Pass->getPassKind());
152}
153
154extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
155  assert(RustPass);
156  Pass *Pass = unwrap(RustPass);
157  PassManagerBase *PMB = unwrap(PMR);
158  PMB->add(Pass);
159}
160
161extern "C"
162void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
163  LLVMPassManagerBuilderRef PMBR,
164  LLVMPassManagerRef PMR
165) {
166  unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
167}
168
169extern "C"
170void LLVMRustAddLastExtensionPasses(
171    LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
172  auto AddExtensionPasses = [Passes, NumPasses](
173      const PassManagerBuilder &Builder, PassManagerBase &PM) {
174    for (size_t I = 0; I < NumPasses; I++) {
175      PM.add(unwrap(Passes[I]));
176    }
177  };
178  // Add the passes to both of the pre-finalization extension points,
179  // so they are run for optimized and non-optimized builds.
180  unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
181                             AddExtensionPasses);
182  unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
183                             AddExtensionPasses);
184}
185
186#ifdef LLVM_COMPONENT_X86
187#define SUBTARGET_X86 SUBTARGET(X86)
188#else
189#define SUBTARGET_X86
190#endif
191
192#ifdef LLVM_COMPONENT_ARM
193#define SUBTARGET_ARM SUBTARGET(ARM)
194#else
195#define SUBTARGET_ARM
196#endif
197
198#ifdef LLVM_COMPONENT_AARCH64
199#define SUBTARGET_AARCH64 SUBTARGET(AArch64)
200#else
201#define SUBTARGET_AARCH64
202#endif
203
204#ifdef LLVM_COMPONENT_MIPS
205#define SUBTARGET_MIPS SUBTARGET(Mips)
206#else
207#define SUBTARGET_MIPS
208#endif
209
210#ifdef LLVM_COMPONENT_POWERPC
211#define SUBTARGET_PPC SUBTARGET(PPC)
212#else
213#define SUBTARGET_PPC
214#endif
215
216#ifdef LLVM_COMPONENT_SYSTEMZ
217#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
218#else
219#define SUBTARGET_SYSTEMZ
220#endif
221
222#ifdef LLVM_COMPONENT_MSP430
223#define SUBTARGET_MSP430 SUBTARGET(MSP430)
224#else
225#define SUBTARGET_MSP430
226#endif
227
228#ifdef LLVM_COMPONENT_RISCV
229#define SUBTARGET_RISCV SUBTARGET(RISCV)
230#else
231#define SUBTARGET_RISCV
232#endif
233
234#ifdef LLVM_COMPONENT_SPARC
235#define SUBTARGET_SPARC SUBTARGET(Sparc)
236#else
237#define SUBTARGET_SPARC
238#endif
239
240#ifdef LLVM_COMPONENT_HEXAGON
241#define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
242#else
243#define SUBTARGET_HEXAGON
244#endif
245
246#define GEN_SUBTARGETS                                                         \
247  SUBTARGET_X86                                                                \
248  SUBTARGET_ARM                                                                \
249  SUBTARGET_AARCH64                                                            \
250  SUBTARGET_MIPS                                                               \
251  SUBTARGET_PPC                                                                \
252  SUBTARGET_SYSTEMZ                                                            \
253  SUBTARGET_MSP430                                                             \
254  SUBTARGET_SPARC                                                              \
255  SUBTARGET_HEXAGON                                                            \
256  SUBTARGET_RISCV                                                              \
257
258#define SUBTARGET(x)                                                           \
259  namespace llvm {                                                             \
260  extern const SubtargetFeatureKV x##FeatureKV[];                              \
261  extern const SubtargetFeatureKV x##SubTypeKV[];                              \
262  }
263
264GEN_SUBTARGETS
265#undef SUBTARGET
266
267extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
268                                   const char *Feature) {
269  TargetMachine *Target = unwrap(TM);
270  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
271  return MCInfo->checkFeatures(std::string("+") + Feature);
272}
273
274enum class LLVMRustCodeModel {
275  Other,
276  Small,
277  Kernel,
278  Medium,
279  Large,
280  None,
281};
282
283static CodeModel::Model fromRust(LLVMRustCodeModel Model) {
284  switch (Model) {
285  case LLVMRustCodeModel::Small:
286    return CodeModel::Small;
287  case LLVMRustCodeModel::Kernel:
288    return CodeModel::Kernel;
289  case LLVMRustCodeModel::Medium:
290    return CodeModel::Medium;
291  case LLVMRustCodeModel::Large:
292    return CodeModel::Large;
293  default:
294    report_fatal_error("Bad CodeModel.");
295  }
296}
297
298enum class LLVMRustCodeGenOptLevel {
299  Other,
300  None,
301  Less,
302  Default,
303  Aggressive,
304};
305
306static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
307  switch (Level) {
308  case LLVMRustCodeGenOptLevel::None:
309    return CodeGenOpt::None;
310  case LLVMRustCodeGenOptLevel::Less:
311    return CodeGenOpt::Less;
312  case LLVMRustCodeGenOptLevel::Default:
313    return CodeGenOpt::Default;
314  case LLVMRustCodeGenOptLevel::Aggressive:
315    return CodeGenOpt::Aggressive;
316  default:
317    report_fatal_error("Bad CodeGenOptLevel.");
318  }
319}
320
321enum class LLVMRustPassBuilderOptLevel {
322  O0,
323  O1,
324  O2,
325  O3,
326  Os,
327  Oz,
328};
329
330static PassBuilder::OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
331  switch (Level) {
332  case LLVMRustPassBuilderOptLevel::O0:
333    return PassBuilder::O0;
334  case LLVMRustPassBuilderOptLevel::O1:
335    return PassBuilder::O1;
336  case LLVMRustPassBuilderOptLevel::O2:
337    return PassBuilder::O2;
338  case LLVMRustPassBuilderOptLevel::O3:
339    return PassBuilder::O3;
340  case LLVMRustPassBuilderOptLevel::Os:
341    return PassBuilder::Os;
342  case LLVMRustPassBuilderOptLevel::Oz:
343    return PassBuilder::Oz;
344  default:
345    report_fatal_error("Bad PassBuilderOptLevel.");
346  }
347}
348
349enum class LLVMRustRelocMode {
350  Default,
351  Static,
352  PIC,
353  DynamicNoPic,
354  ROPI,
355  RWPI,
356  ROPIRWPI,
357};
358
359static Optional<Reloc::Model> fromRust(LLVMRustRelocMode RustReloc) {
360  switch (RustReloc) {
361  case LLVMRustRelocMode::Default:
362    return None;
363  case LLVMRustRelocMode::Static:
364    return Reloc::Static;
365  case LLVMRustRelocMode::PIC:
366    return Reloc::PIC_;
367  case LLVMRustRelocMode::DynamicNoPic:
368    return Reloc::DynamicNoPIC;
369  case LLVMRustRelocMode::ROPI:
370    return Reloc::ROPI;
371  case LLVMRustRelocMode::RWPI:
372    return Reloc::RWPI;
373  case LLVMRustRelocMode::ROPIRWPI:
374    return Reloc::ROPI_RWPI;
375  }
376  report_fatal_error("Bad RelocModel.");
377}
378
379#ifdef LLVM_RUSTLLVM
380/// getLongestEntryLength - Return the length of the longest entry in the table.
381template<typename KV>
382static size_t getLongestEntryLength(ArrayRef<KV> Table) {
383  size_t MaxLen = 0;
384  for (auto &I : Table)
385    MaxLen = std::max(MaxLen, std::strlen(I.Key));
386  return MaxLen;
387}
388
389extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
390  const TargetMachine *Target = unwrap(TM);
391  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
392  const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
393  const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
394  const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
395  unsigned MaxCPULen = getLongestEntryLength(CPUTable);
396
397  printf("Available CPUs for this target:\n");
398  if (HostArch == TargetArch) {
399    const StringRef HostCPU = sys::getHostCPUName();
400    printf("    %-*s - Select the CPU of the current host (currently %.*s).\n",
401      MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
402  }
403  for (auto &CPU : CPUTable)
404    printf("    %-*s\n", MaxCPULen, CPU.Key);
405  printf("\n");
406}
407
408extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) {
409  const TargetMachine *Target = unwrap(TM);
410  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
411  const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
412  unsigned MaxFeatLen = getLongestEntryLength(FeatTable);
413
414  printf("Available features for this target:\n");
415  for (auto &Feature : FeatTable)
416    printf("    %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc);
417  printf("\n");
418
419  printf("Use +feature to enable a feature, or -feature to disable it.\n"
420         "For example, rustc -C -target-cpu=mycpu -C "
421         "target-feature=+feature1,-feature2\n\n");
422}
423
424#else
425
426extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
427  printf("Target CPU help is not supported by this LLVM version.\n\n");
428}
429
430extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) {
431  printf("Target features help is not supported by this LLVM version.\n\n");
432}
433#endif
434
435extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
436  StringRef Name = sys::getHostCPUName();
437  *len = Name.size();
438  return Name.data();
439}
440
441extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
442    const char *TripleStr, const char *CPU, const char *Feature,
443    const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocMode RustReloc,
444    LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
445    bool PositionIndependentExecutable, bool FunctionSections,
446    bool DataSections,
447    bool TrapUnreachable,
448    bool Singlethread,
449    bool AsmComments,
450    bool EmitStackSizeSection,
451    bool RelaxELFRelocations) {
452
453  auto OptLevel = fromRust(RustOptLevel);
454  auto RM = fromRust(RustReloc);
455
456  std::string Error;
457  Triple Trip(Triple::normalize(TripleStr));
458  const llvm::Target *TheTarget =
459      TargetRegistry::lookupTarget(Trip.getTriple(), Error);
460  if (TheTarget == nullptr) {
461    LLVMRustSetLastError(Error.c_str());
462    return nullptr;
463  }
464
465  TargetOptions Options;
466
467  Options.FloatABIType = FloatABI::Default;
468  if (UseSoftFloat) {
469    Options.FloatABIType = FloatABI::Soft;
470  }
471  Options.DataSections = DataSections;
472  Options.FunctionSections = FunctionSections;
473  Options.MCOptions.AsmVerbose = AsmComments;
474  Options.MCOptions.PreserveAsmComments = AsmComments;
475  Options.MCOptions.ABIName = ABIStr;
476  Options.RelaxELFRelocations = RelaxELFRelocations;
477
478  if (TrapUnreachable) {
479    // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
480    // This limits the extent of possible undefined behavior in some cases, as
481    // it prevents control flow from "falling through" into whatever code
482    // happens to be laid out next in memory.
483    Options.TrapUnreachable = true;
484  }
485
486  if (Singlethread) {
487    Options.ThreadModel = ThreadModel::Single;
488  }
489
490  Options.EmitStackSizeSection = EmitStackSizeSection;
491
492  Optional<CodeModel::Model> CM;
493  if (RustCM != LLVMRustCodeModel::None)
494    CM = fromRust(RustCM);
495  TargetMachine *TM = TheTarget->createTargetMachine(
496      Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
497  return wrap(TM);
498}
499
500extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
501  delete unwrap(TM);
502}
503
504extern "C" void LLVMRustConfigurePassManagerBuilder(
505    LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
506    bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
507    const char* PGOGenPath, const char* PGOUsePath) {
508  unwrap(PMBR)->MergeFunctions = MergeFunctions;
509  unwrap(PMBR)->SLPVectorize = SLPVectorize;
510  unwrap(PMBR)->OptLevel = fromRust(OptLevel);
511  unwrap(PMBR)->LoopVectorize = LoopVectorize;
512  unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
513
514  if (PGOGenPath) {
515    assert(!PGOUsePath);
516    unwrap(PMBR)->EnablePGOInstrGen = true;
517    unwrap(PMBR)->PGOInstrGen = PGOGenPath;
518  }
519  if (PGOUsePath) {
520    assert(!PGOGenPath);
521    unwrap(PMBR)->PGOInstrUse = PGOUsePath;
522  }
523}
524
525// Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
526// field of a PassManagerBuilder, we expose our own method of doing so.
527extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
528                                              LLVMModuleRef M,
529                                              bool DisableSimplifyLibCalls) {
530  Triple TargetTriple(unwrap(M)->getTargetTriple());
531  TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
532  if (DisableSimplifyLibCalls)
533    TLI->disableAllFunctions();
534  unwrap(PMBR)->LibraryInfo = TLI;
535}
536
537// Unfortunately, the LLVM C API doesn't provide a way to create the
538// TargetLibraryInfo pass, so we use this method to do so.
539extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
540                                       bool DisableSimplifyLibCalls) {
541  Triple TargetTriple(unwrap(M)->getTargetTriple());
542  TargetLibraryInfoImpl TLII(TargetTriple);
543  if (DisableSimplifyLibCalls)
544    TLII.disableAllFunctions();
545  unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
546}
547
548// Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
549// all the functions in a module, so we do that manually here. You'll find
550// similar code in clang's BackendUtil.cpp file.
551extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
552                                               LLVMModuleRef M) {
553  llvm::legacy::FunctionPassManager *P =
554      unwrap<llvm::legacy::FunctionPassManager>(PMR);
555  P->doInitialization();
556
557  // Upgrade all calls to old intrinsics first.
558  for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
559    UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
560
561  for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
562       ++I)
563    if (!I->isDeclaration())
564      P->run(*I);
565
566  P->doFinalization();
567}
568
569extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
570  // Initializing the command-line options more than once is not allowed. So,
571  // check if they've already been initialized.  (This could happen if we're
572  // being called from rustpkg, for example). If the arguments change, then
573  // that's just kinda unfortunate.
574  static bool Initialized = false;
575  if (Initialized)
576    return;
577  Initialized = true;
578  cl::ParseCommandLineOptions(Argc, Argv);
579}
580
581enum class LLVMRustFileType {
582  Other,
583  AssemblyFile,
584  ObjectFile,
585};
586
587#if LLVM_VERSION_GE(10, 0)
588static CodeGenFileType fromRust(LLVMRustFileType Type) {
589  switch (Type) {
590  case LLVMRustFileType::AssemblyFile:
591    return CGFT_AssemblyFile;
592  case LLVMRustFileType::ObjectFile:
593    return CGFT_ObjectFile;
594  default:
595    report_fatal_error("Bad FileType.");
596  }
597}
598#else
599static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) {
600  switch (Type) {
601  case LLVMRustFileType::AssemblyFile:
602    return TargetMachine::CGFT_AssemblyFile;
603  case LLVMRustFileType::ObjectFile:
604    return TargetMachine::CGFT_ObjectFile;
605  default:
606    report_fatal_error("Bad FileType.");
607  }
608}
609#endif
610
611extern "C" LLVMRustResult
612LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
613                        LLVMModuleRef M, const char *Path,
614                        LLVMRustFileType RustFileType) {
615  llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
616  auto FileType = fromRust(RustFileType);
617
618  std::string ErrorInfo;
619  std::error_code EC;
620  raw_fd_ostream OS(Path, EC, sys::fs::F_None);
621  if (EC)
622    ErrorInfo = EC.message();
623  if (ErrorInfo != "") {
624    LLVMRustSetLastError(ErrorInfo.c_str());
625    return LLVMRustResult::Failure;
626  }
627
628  buffer_ostream BOS(OS);
629  unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
630  PM->run(*unwrap(M));
631
632  // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
633  // stream (OS), so the only real safe place to delete this is here? Don't we
634  // wish this was written in Rust?
635  LLVMDisposePassManager(PMR);
636  return LLVMRustResult::Success;
637}
638
639extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
640                                                      const char*,      // pass name
641                                                      const char*);     // IR name
642extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
643
644#if LLVM_VERSION_GE(9, 0)
645
646std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
647  if (any_isa<const Module *>(WrappedIr))
648    return any_cast<const Module *>(WrappedIr)->getName().str();
649  if (any_isa<const Function *>(WrappedIr))
650    return any_cast<const Function *>(WrappedIr)->getName().str();
651  if (any_isa<const Loop *>(WrappedIr))
652    return any_cast<const Loop *>(WrappedIr)->getName().str();
653  if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
654    return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
655  return "<UNKNOWN>";
656}
657
658
659void LLVMSelfProfileInitializeCallbacks(
660    PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
661    LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
662    LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
663  PIC.registerBeforePassCallback([LlvmSelfProfiler, BeforePassCallback](
664                                     StringRef Pass, llvm::Any Ir) {
665    std::string PassName = Pass.str();
666    std::string IrName = LLVMRustwrappedIrGetName(Ir);
667    BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
668    return true;
669  });
670
671  PIC.registerAfterPassCallback(
672      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
673        AfterPassCallback(LlvmSelfProfiler);
674      });
675
676  PIC.registerAfterPassInvalidatedCallback(
677      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass) {
678        AfterPassCallback(LlvmSelfProfiler);
679      });
680
681  PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
682                                         StringRef Pass, llvm::Any Ir) {
683    std::string PassName = Pass.str();
684    std::string IrName = LLVMRustwrappedIrGetName(Ir);
685    BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
686  });
687
688  PIC.registerAfterAnalysisCallback(
689      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
690        AfterPassCallback(LlvmSelfProfiler);
691      });
692}
693#endif
694
695enum class LLVMRustOptStage {
696  PreLinkNoLTO,
697  PreLinkThinLTO,
698  PreLinkFatLTO,
699  ThinLTO,
700  FatLTO,
701};
702
703struct LLVMRustSanitizerOptions {
704  bool SanitizeMemory;
705  bool SanitizeThread;
706  bool SanitizeAddress;
707  bool SanitizeRecover;
708  int SanitizeMemoryTrackOrigins;
709};
710
711extern "C" void
712LLVMRustOptimizeWithNewPassManager(
713    LLVMModuleRef ModuleRef,
714    LLVMTargetMachineRef TMRef,
715    LLVMRustPassBuilderOptLevel OptLevelRust,
716    LLVMRustOptStage OptStage,
717    bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
718    bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
719    bool DisableSimplifyLibCalls,
720    LLVMRustSanitizerOptions *SanitizerOptions,
721    const char *PGOGenPath, const char *PGOUsePath,
722    void* LlvmSelfProfiler,
723    LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
724    LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
725#if LLVM_VERSION_GE(9, 0)
726  Module *TheModule = unwrap(ModuleRef);
727  TargetMachine *TM = unwrap(TMRef);
728  PassBuilder::OptimizationLevel OptLevel = fromRust(OptLevelRust);
729
730  // FIXME: MergeFunctions is not supported by NewPM yet.
731  (void) MergeFunctions;
732
733  PipelineTuningOptions PTO;
734  PTO.LoopUnrolling = UnrollLoops;
735  PTO.LoopInterleaving = UnrollLoops;
736  PTO.LoopVectorization = LoopVectorize;
737  PTO.SLPVectorization = SLPVectorize;
738
739  PassInstrumentationCallbacks PIC;
740  StandardInstrumentations SI;
741  SI.registerCallbacks(PIC);
742
743  if (LlvmSelfProfiler){
744    LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
745  }
746
747  Optional<PGOOptions> PGOOpt;
748  if (PGOGenPath) {
749    assert(!PGOUsePath);
750    PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr);
751  } else if (PGOUsePath) {
752    assert(!PGOGenPath);
753    PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse);
754  }
755
756  PassBuilder PB(TM, PTO, PGOOpt, &PIC);
757
758  // FIXME: We may want to expose this as an option.
759  bool DebugPassManager = false;
760  LoopAnalysisManager LAM(DebugPassManager);
761  FunctionAnalysisManager FAM(DebugPassManager);
762  CGSCCAnalysisManager CGAM(DebugPassManager);
763  ModuleAnalysisManager MAM(DebugPassManager);
764
765  FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
766
767  Triple TargetTriple(TheModule->getTargetTriple());
768  std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
769  if (DisableSimplifyLibCalls)
770    TLII->disableAllFunctions();
771  FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
772
773  PB.registerModuleAnalyses(MAM);
774  PB.registerCGSCCAnalyses(CGAM);
775  PB.registerFunctionAnalyses(FAM);
776  PB.registerLoopAnalyses(LAM);
777  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
778
779  // We manually collect pipeline callbacks so we can apply them at O0, where the
780  // PassBuilder does not create a pipeline.
781  std::vector<std::function<void(ModulePassManager &)>> PipelineStartEPCallbacks;
782  std::vector<std::function<void(FunctionPassManager &, PassBuilder::OptimizationLevel)>>
783      OptimizerLastEPCallbacks;
784
785  if (VerifyIR) {
786    PipelineStartEPCallbacks.push_back([VerifyIR](ModulePassManager &MPM) {
787        MPM.addPass(VerifierPass());
788    });
789  }
790
791  if (SanitizerOptions) {
792    if (SanitizerOptions->SanitizeMemory) {
793      MemorySanitizerOptions Options(
794          SanitizerOptions->SanitizeMemoryTrackOrigins,
795          SanitizerOptions->SanitizeRecover,
796          /*CompileKernel=*/false);
797#if LLVM_VERSION_GE(10, 0)
798      PipelineStartEPCallbacks.push_back([Options](ModulePassManager &MPM) {
799        MPM.addPass(MemorySanitizerPass(Options));
800      });
801#endif
802      OptimizerLastEPCallbacks.push_back(
803        [Options](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
804          FPM.addPass(MemorySanitizerPass(Options));
805        }
806      );
807    }
808
809    if (SanitizerOptions->SanitizeThread) {
810#if LLVM_VERSION_GE(10, 0)
811      PipelineStartEPCallbacks.push_back([](ModulePassManager &MPM) {
812        MPM.addPass(ThreadSanitizerPass());
813      });
814#endif
815      OptimizerLastEPCallbacks.push_back(
816        [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
817          FPM.addPass(ThreadSanitizerPass());
818        }
819      );
820    }
821
822    if (SanitizerOptions->SanitizeAddress) {
823      PipelineStartEPCallbacks.push_back([&](ModulePassManager &MPM) {
824        MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
825      });
826      OptimizerLastEPCallbacks.push_back(
827        [SanitizerOptions](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
828          FPM.addPass(AddressSanitizerPass(
829              /*CompileKernel=*/false, SanitizerOptions->SanitizeRecover,
830              /*UseAfterScope=*/true));
831        }
832      );
833      PipelineStartEPCallbacks.push_back(
834        [SanitizerOptions](ModulePassManager &MPM) {
835          MPM.addPass(ModuleAddressSanitizerPass(
836              /*CompileKernel=*/false, SanitizerOptions->SanitizeRecover));
837        }
838      );
839    }
840  }
841
842  ModulePassManager MPM(DebugPassManager);
843  if (!NoPrepopulatePasses) {
844    if (OptLevel == PassBuilder::O0) {
845      for (const auto &C : PipelineStartEPCallbacks)
846        C(MPM);
847
848      if (!OptimizerLastEPCallbacks.empty()) {
849        FunctionPassManager FPM(DebugPassManager);
850        for (const auto &C : OptimizerLastEPCallbacks)
851          C(FPM, OptLevel);
852        MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
853      }
854
855      MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/false));
856
857#if LLVM_VERSION_GE(10, 0)
858      if (PGOOpt) {
859        PB.addPGOInstrPassesForO0(
860            MPM, DebugPassManager, PGOOpt->Action == PGOOptions::IRInstr,
861            /*IsCS=*/false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
862      }
863#endif
864    } else {
865      for (const auto &C : PipelineStartEPCallbacks)
866        PB.registerPipelineStartEPCallback(C);
867      if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
868        for (const auto &C : OptimizerLastEPCallbacks)
869          PB.registerOptimizerLastEPCallback(C);
870      }
871
872      switch (OptStage) {
873      case LLVMRustOptStage::PreLinkNoLTO:
874        MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
875        break;
876      case LLVMRustOptStage::PreLinkThinLTO:
877        MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
878        if (!OptimizerLastEPCallbacks.empty()) {
879          FunctionPassManager FPM(DebugPassManager);
880          for (const auto &C : OptimizerLastEPCallbacks)
881            C(FPM, OptLevel);
882          MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
883        }
884        break;
885      case LLVMRustOptStage::PreLinkFatLTO:
886        MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
887        break;
888      case LLVMRustOptStage::ThinLTO:
889        // FIXME: Does it make sense to pass the ModuleSummaryIndex?
890        // It only seems to be needed for C++ specific optimizations.
891        MPM = PB.buildThinLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
892        break;
893      case LLVMRustOptStage::FatLTO:
894        MPM = PB.buildLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
895        break;
896      }
897    }
898  }
899
900  if (UseThinLTOBuffers) {
901    MPM.addPass(CanonicalizeAliasesPass());
902    MPM.addPass(NameAnonGlobalPass());
903  }
904
905  // Upgrade all calls to old intrinsics first.
906  for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
907    UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
908
909  MPM.run(*TheModule, MAM);
910#else
911  // The new pass manager has been available for a long time,
912  // but we don't bother supporting it on old LLVM versions.
913  report_fatal_error("New pass manager only supported since LLVM 9");
914#endif
915}
916
917// Callback to demangle function name
918// Parameters:
919// * name to be demangled
920// * name len
921// * output buffer
922// * output buffer len
923// Returns len of demangled string, or 0 if demangle failed.
924typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
925
926
927namespace {
928
929class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
930  DemangleFn Demangle;
931  std::vector<char> Buf;
932
933public:
934  RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
935
936  // Return empty string if demangle failed
937  // or if name does not need to be demangled
938  StringRef CallDemangle(StringRef name) {
939    if (!Demangle) {
940      return StringRef();
941    }
942
943    if (Buf.size() < name.size() * 2) {
944      // Semangled name usually shorter than mangled,
945      // but allocate twice as much memory just in case
946      Buf.resize(name.size() * 2);
947    }
948
949    auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
950    if (!R) {
951      // Demangle failed.
952      return StringRef();
953    }
954
955    auto Demangled = StringRef(Buf.data(), R);
956    if (Demangled == name) {
957      // Do not print anything if demangled name is equal to mangled.
958      return StringRef();
959    }
960
961    return Demangled;
962  }
963
964  void emitFunctionAnnot(const Function *F,
965                         formatted_raw_ostream &OS) override {
966    StringRef Demangled = CallDemangle(F->getName());
967    if (Demangled.empty()) {
968        return;
969    }
970
971    OS << "; " << Demangled << "\n";
972  }
973
974  void emitInstructionAnnot(const Instruction *I,
975                            formatted_raw_ostream &OS) override {
976    const char *Name;
977    const Value *Value;
978    if (const CallInst *CI = dyn_cast<CallInst>(I)) {
979      Name = "call";
980      Value = CI->getCalledValue();
981    } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
982      Name = "invoke";
983      Value = II->getCalledValue();
984    } else {
985      // Could demangle more operations, e. g.
986      // `store %place, @function`.
987      return;
988    }
989
990    if (!Value->hasName()) {
991      return;
992    }
993
994    StringRef Demangled = CallDemangle(Value->getName());
995    if (Demangled.empty()) {
996      return;
997    }
998
999    OS << "; " << Name << " " << Demangled << "\n";
1000  }
1001};
1002
1003} // namespace
1004
1005extern "C" LLVMRustResult
1006LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
1007  std::string ErrorInfo;
1008  std::error_code EC;
1009  raw_fd_ostream OS(Path, EC, sys::fs::F_None);
1010  if (EC)
1011    ErrorInfo = EC.message();
1012  if (ErrorInfo != "") {
1013    LLVMRustSetLastError(ErrorInfo.c_str());
1014    return LLVMRustResult::Failure;
1015  }
1016
1017  RustAssemblyAnnotationWriter AAW(Demangle);
1018  formatted_raw_ostream FOS(OS);
1019  unwrap(M)->print(FOS, &AAW);
1020
1021  return LLVMRustResult::Success;
1022}
1023
1024extern "C" void LLVMRustPrintPasses() {
1025  LLVMInitializePasses();
1026  struct MyListener : PassRegistrationListener {
1027    void passEnumerate(const PassInfo *Info) {
1028      StringRef PassArg = Info->getPassArgument();
1029      StringRef PassName = Info->getPassName();
1030      if (!PassArg.empty()) {
1031        // These unsigned->signed casts could theoretically overflow, but
1032        // realistically never will (and even if, the result is implementation
1033        // defined rather plain UB).
1034        printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1035               (int)PassName.size(), PassName.data());
1036      }
1037    }
1038  } Listener;
1039
1040  PassRegistry *PR = PassRegistry::getPassRegistry();
1041  PR->enumerateWith(&Listener);
1042}
1043
1044extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
1045                                            bool AddLifetimes) {
1046  unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
1047}
1048
1049extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1050                                           size_t Len) {
1051  llvm::legacy::PassManager passes;
1052
1053  auto PreserveFunctions = [=](const GlobalValue &GV) {
1054    for (size_t I = 0; I < Len; I++) {
1055      if (GV.getName() == Symbols[I]) {
1056        return true;
1057      }
1058    }
1059    return false;
1060  };
1061
1062  passes.add(llvm::createInternalizePass(PreserveFunctions));
1063
1064  passes.run(*unwrap(M));
1065}
1066
1067extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) {
1068  for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E;
1069       ++GV) {
1070    GV->setDoesNotThrow();
1071    Function *F = dyn_cast<Function>(GV);
1072    if (F == nullptr)
1073      continue;
1074
1075    for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
1076      for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) {
1077        if (isa<InvokeInst>(I)) {
1078          InvokeInst *CI = cast<InvokeInst>(I);
1079          CI->setDoesNotThrow();
1080        }
1081      }
1082    }
1083  }
1084}
1085
1086extern "C" void
1087LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1088                                       LLVMTargetMachineRef TMR) {
1089  TargetMachine *Target = unwrap(TMR);
1090  unwrap(Module)->setDataLayout(Target->createDataLayout());
1091}
1092
1093extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1094  unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1095}
1096
1097extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1098  unwrap(M)->setPIELevel(PIELevel::Level::Large);
1099}
1100
1101// Here you'll find an implementation of ThinLTO as used by the Rust compiler
1102// right now. This ThinLTO support is only enabled on "recent ish" versions of
1103// LLVM, and otherwise it's just blanket rejected from other compilers.
1104//
1105// Most of this implementation is straight copied from LLVM. At the time of
1106// this writing it wasn't *quite* suitable to reuse more code from upstream
1107// for our purposes, but we should strive to upstream this support once it's
1108// ready to go! I figure we may want a bit of testing locally first before
1109// sending this upstream to LLVM. I hear though they're quite eager to receive
1110// feedback like this!
1111//
1112// If you're reading this code and wondering "what in the world" or you're
1113// working "good lord by LLVM upgrade is *still* failing due to these bindings"
1114// then fear not! (ok maybe fear a little). All code here is mostly based
1115// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1116//
1117// You'll find that the general layout here roughly corresponds to the `run`
1118// method in that file as well as `ProcessThinLTOModule`. Functions are
1119// specifically commented below as well, but if you're updating this code
1120// or otherwise trying to understand it, the LLVM source will be useful in
1121// interpreting the mysteries within.
1122//
1123// Otherwise I'll apologize in advance, it probably requires a relatively
1124// significant investment on your part to "truly understand" what's going on
1125// here. Not saying I do myself, but it took me awhile staring at LLVM's source
1126// and various online resources about ThinLTO to make heads or tails of all
1127// this.
1128
1129// This is a shared data structure which *must* be threadsafe to share
1130// read-only amongst threads. This also corresponds basically to the arguments
1131// of the `ProcessThinLTOModule` function in the LLVM source.
1132struct LLVMRustThinLTOData {
1133  // The combined index that is the global analysis over all modules we're
1134  // performing ThinLTO for. This is mostly managed by LLVM.
1135  ModuleSummaryIndex Index;
1136
1137  // All modules we may look at, stored as in-memory serialized versions. This
1138  // is later used when inlining to ensure we can extract any module to inline
1139  // from.
1140  StringMap<MemoryBufferRef> ModuleMap;
1141
1142  // A set that we manage of everything we *don't* want internalized. Note that
1143  // this includes all transitive references right now as well, but it may not
1144  // always!
1145  DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1146
1147  // Not 100% sure what these are, but they impact what's internalized and
1148  // what's inlined across modules, I believe.
1149  StringMap<FunctionImporter::ImportMapTy> ImportLists;
1150  StringMap<FunctionImporter::ExportSetTy> ExportLists;
1151  StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1152
1153  LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1154};
1155
1156// Just an argument to the `LLVMRustCreateThinLTOData` function below.
1157struct LLVMRustThinLTOModule {
1158  const char *identifier;
1159  const char *data;
1160  size_t len;
1161};
1162
1163// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1164// does.
1165static const GlobalValueSummary *
1166getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1167  auto StrongDefForLinker = llvm::find_if(
1168      GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1169        auto Linkage = Summary->linkage();
1170        return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1171               !GlobalValue::isWeakForLinker(Linkage);
1172      });
1173  if (StrongDefForLinker != GVSummaryList.end())
1174    return StrongDefForLinker->get();
1175
1176  auto FirstDefForLinker = llvm::find_if(
1177      GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1178        auto Linkage = Summary->linkage();
1179        return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1180      });
1181  if (FirstDefForLinker == GVSummaryList.end())
1182    return nullptr;
1183  return FirstDefForLinker->get();
1184}
1185
1186// The main entry point for creating the global ThinLTO analysis. The structure
1187// here is basically the same as before threads are spawned in the `run`
1188// function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1189extern "C" LLVMRustThinLTOData*
1190LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1191                          int num_modules,
1192                          const char **preserved_symbols,
1193                          int num_symbols) {
1194#if LLVM_VERSION_GE(10, 0)
1195  auto Ret = std::make_unique<LLVMRustThinLTOData>();
1196#else
1197  auto Ret = llvm::make_unique<LLVMRustThinLTOData>();
1198#endif
1199
1200  // Load each module's summary and merge it into one combined index
1201  for (int i = 0; i < num_modules; i++) {
1202    auto module = &modules[i];
1203    StringRef buffer(module->data, module->len);
1204    MemoryBufferRef mem_buffer(buffer, module->identifier);
1205
1206    Ret->ModuleMap[module->identifier] = mem_buffer;
1207
1208    if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1209      LLVMRustSetLastError(toString(std::move(Err)).c_str());
1210      return nullptr;
1211    }
1212  }
1213
1214  // Collect for each module the list of function it defines (GUID -> Summary)
1215  Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1216
1217  // Convert the preserved symbols set from string to GUID, this is then needed
1218  // for internalization.
1219  for (int i = 0; i < num_symbols; i++) {
1220    auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1221    Ret->GUIDPreservedSymbols.insert(GUID);
1222  }
1223
1224  // Collect the import/export lists for all modules from the call-graph in the
1225  // combined index
1226  //
1227  // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1228  auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1229    return PrevailingType::Unknown;
1230  };
1231  // We don't have a complete picture in our use of ThinLTO, just our immediate
1232  // crate, so we need `ImportEnabled = false` to limit internalization.
1233  // Otherwise, we sometimes lose `static` values -- see #60184.
1234  computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1235                                  deadIsPrevailing, /* ImportEnabled = */ false);
1236  ComputeCrossModuleImport(
1237    Ret->Index,
1238    Ret->ModuleToDefinedGVSummaries,
1239    Ret->ImportLists,
1240    Ret->ExportLists
1241  );
1242
1243  // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1244  // impacts the caching.
1245  //
1246  // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1247  // being lifted from `lib/LTO/LTO.cpp` as well
1248  StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1249  DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1250  for (auto &I : Ret->Index) {
1251    if (I.second.SummaryList.size() > 1)
1252      PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1253  }
1254  auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1255    const auto &Prevailing = PrevailingCopy.find(GUID);
1256    if (Prevailing == PrevailingCopy.end())
1257      return true;
1258    return Prevailing->second == S;
1259  };
1260  auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1261                              GlobalValue::GUID GUID,
1262                              GlobalValue::LinkageTypes NewLinkage) {
1263    ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1264  };
1265#if LLVM_VERSION_GE(9, 0)
1266  thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage,
1267                                  Ret->GUIDPreservedSymbols);
1268#else
1269  thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage);
1270#endif
1271
1272  // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1273  // callback below. This callback below will dictate the linkage for all
1274  // summaries in the index, and we basically just only want to ensure that dead
1275  // symbols are internalized. Otherwise everything that's already external
1276  // linkage will stay as external, and internal will stay as internal.
1277  std::set<GlobalValue::GUID> ExportedGUIDs;
1278  for (auto &List : Ret->Index) {
1279    for (auto &GVS: List.second.SummaryList) {
1280      if (GlobalValue::isLocalLinkage(GVS->linkage()))
1281        continue;
1282      auto GUID = GVS->getOriginalName();
1283      if (GVS->flags().Live)
1284        ExportedGUIDs.insert(GUID);
1285    }
1286  }
1287#if LLVM_VERSION_GE(10, 0)
1288  auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1289    const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1290    return (ExportList != Ret->ExportLists.end() &&
1291      ExportList->second.count(VI)) ||
1292      ExportedGUIDs.count(VI.getGUID());
1293  };
1294  thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1295#else
1296  auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
1297    const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1298    return (ExportList != Ret->ExportLists.end() &&
1299      ExportList->second.count(GUID)) ||
1300      ExportedGUIDs.count(GUID);
1301  };
1302  thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported);
1303#endif
1304
1305  return Ret.release();
1306}
1307
1308extern "C" void
1309LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1310  delete Data;
1311}
1312
1313// Below are the various passes that happen *per module* when doing ThinLTO.
1314//
1315// In other words, these are the functions that are all run concurrently
1316// with one another, one per module. The passes here correspond to the analysis
1317// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1318// `ProcessThinLTOModule` function. Here they're split up into separate steps
1319// so rustc can save off the intermediate bytecode between each step.
1320
1321extern "C" bool
1322LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1323  Module &Mod = *unwrap(M);
1324  if (renameModuleForThinLTO(Mod, Data->Index)) {
1325    LLVMRustSetLastError("renameModuleForThinLTO failed");
1326    return false;
1327  }
1328  return true;
1329}
1330
1331extern "C" bool
1332LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1333  Module &Mod = *unwrap(M);
1334  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1335  thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1336  return true;
1337}
1338
1339extern "C" bool
1340LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1341  Module &Mod = *unwrap(M);
1342  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1343  thinLTOInternalizeModule(Mod, DefinedGlobals);
1344  return true;
1345}
1346
1347extern "C" bool
1348LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1349  Module &Mod = *unwrap(M);
1350
1351  const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1352  auto Loader = [&](StringRef Identifier) {
1353    const auto &Memory = Data->ModuleMap.lookup(Identifier);
1354    auto &Context = Mod.getContext();
1355    auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1356
1357    if (!MOrErr)
1358      return MOrErr;
1359
1360    // The rest of this closure is a workaround for
1361    // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1362    // we accidentally import wasm custom sections into different modules,
1363    // duplicating them by in the final output artifact.
1364    //
1365    // The issue is worked around here by manually removing the
1366    // `wasm.custom_sections` named metadata node from any imported module. This
1367    // we know isn't used by any optimization pass so there's no need for it to
1368    // be imported.
1369    //
1370    // Note that the metadata is currently lazily loaded, so we materialize it
1371    // here before looking up if there's metadata inside. The `FunctionImporter`
1372    // will immediately materialize metadata anyway after an import, so this
1373    // shouldn't be a perf hit.
1374    if (Error Err = (*MOrErr)->materializeMetadata()) {
1375      Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1376      return Ret;
1377    }
1378
1379    auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1380    if (WasmCustomSections)
1381      WasmCustomSections->eraseFromParent();
1382
1383    return MOrErr;
1384  };
1385  FunctionImporter Importer(Data->Index, Loader);
1386  Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1387  if (!Result) {
1388    LLVMRustSetLastError(toString(Result.takeError()).c_str());
1389    return false;
1390  }
1391  return true;
1392}
1393
1394extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload
1395                                                      const char*, // importing module name
1396                                                      const char*); // imported module name
1397
1398// Calls `module_name_callback` for each module import done by ThinLTO.
1399// The callback is provided with regular null-terminated C strings.
1400extern "C" void
1401LLVMRustGetThinLTOModuleImports(const LLVMRustThinLTOData *data,
1402                                LLVMRustModuleNameCallback module_name_callback,
1403                                void* callback_payload) {
1404  for (const auto& importing_module : data->ImportLists) {
1405    const std::string importing_module_id = importing_module.getKey().str();
1406    const auto& imports = importing_module.getValue();
1407    for (const auto& imported_module : imports) {
1408      const std::string imported_module_id = imported_module.getKey().str();
1409      module_name_callback(callback_payload,
1410                           importing_module_id.c_str(),
1411                           imported_module_id.c_str());
1412    }
1413  }
1414}
1415
1416// This struct and various functions are sort of a hack right now, but the
1417// problem is that we've got in-memory LLVM modules after we generate and
1418// optimize all codegen-units for one compilation in rustc. To be compatible
1419// with the LTO support above we need to serialize the modules plus their
1420// ThinLTO summary into memory.
1421//
1422// This structure is basically an owned version of a serialize module, with
1423// a ThinLTO summary attached.
1424struct LLVMRustThinLTOBuffer {
1425  std::string data;
1426};
1427
1428extern "C" LLVMRustThinLTOBuffer*
1429LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
1430#if LLVM_VERSION_GE(10, 0)
1431  auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1432#else
1433  auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>();
1434#endif
1435  {
1436    raw_string_ostream OS(Ret->data);
1437    {
1438      legacy::PassManager PM;
1439      PM.add(createWriteThinLTOBitcodePass(OS));
1440      PM.run(*unwrap(M));
1441    }
1442  }
1443  return Ret.release();
1444}
1445
1446extern "C" void
1447LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1448  delete Buffer;
1449}
1450
1451extern "C" const void*
1452LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1453  return Buffer->data.data();
1454}
1455
1456extern "C" size_t
1457LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1458  return Buffer->data.length();
1459}
1460
1461// This is what we used to parse upstream bitcode for actual ThinLTO
1462// processing.  We'll call this once per module optimized through ThinLTO, and
1463// it'll be called concurrently on many threads.
1464extern "C" LLVMModuleRef
1465LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1466                           const char *data,
1467                           size_t len,
1468                           const char *identifier) {
1469  StringRef Data(data, len);
1470  MemoryBufferRef Buffer(Data, identifier);
1471  unwrap(Context)->enableDebugTypeODRUniquing();
1472  Expected<std::unique_ptr<Module>> SrcOrError =
1473      parseBitcodeFile(Buffer, *unwrap(Context));
1474  if (!SrcOrError) {
1475    LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1476    return nullptr;
1477  }
1478  return wrap(std::move(*SrcOrError).release());
1479}
1480
1481// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1482// the comment in `back/lto.rs` for why this exists.
1483extern "C" void
1484LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1485                                DICompileUnit **A,
1486                                DICompileUnit **B) {
1487  Module *M = unwrap(Mod);
1488  DICompileUnit **Cur = A;
1489  DICompileUnit **Next = B;
1490  for (DICompileUnit *CU : M->debug_compile_units()) {
1491    *Cur = CU;
1492    Cur = Next;
1493    Next = nullptr;
1494    if (Cur == nullptr)
1495      break;
1496  }
1497}
1498
1499// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1500// the comment in `back/lto.rs` for why this exists.
1501extern "C" void
1502LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1503  Module *M = unwrap(Mod);
1504
1505  // If the original source module didn't have a `DICompileUnit` then try to
1506  // merge all the existing compile units. If there aren't actually any though
1507  // then there's not much for us to do so return.
1508  if (Unit == nullptr) {
1509    for (DICompileUnit *CU : M->debug_compile_units()) {
1510      Unit = CU;
1511      break;
1512    }
1513    if (Unit == nullptr)
1514      return;
1515  }
1516
1517  // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1518  // process it recursively. Note that we specifically iterate over instructions
1519  // to ensure we feed everything into it.
1520  DebugInfoFinder Finder;
1521  Finder.processModule(*M);
1522  for (Function &F : M->functions()) {
1523    for (auto &FI : F) {
1524      for (Instruction &BI : FI) {
1525        if (auto Loc = BI.getDebugLoc())
1526          Finder.processLocation(*M, Loc);
1527        if (auto DVI = dyn_cast<DbgValueInst>(&BI))
1528          Finder.processValue(*M, DVI);
1529        if (auto DDI = dyn_cast<DbgDeclareInst>(&BI))
1530          Finder.processDeclare(*M, DDI);
1531      }
1532    }
1533  }
1534
1535  // After we've found all our debuginfo, rewrite all subprograms to point to
1536  // the same `DICompileUnit`.
1537  for (auto &F : Finder.subprograms()) {
1538    F->replaceUnit(Unit);
1539  }
1540
1541  // Erase any other references to other `DICompileUnit` instances, the verifier
1542  // will later ensure that we don't actually have any other stale references to
1543  // worry about.
1544  auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1545  MD->clearOperands();
1546  MD->addOperand(Unit);
1547}