fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #71868 - Mark-Simulacrum:stable-1.43.1, r=pietroalbini8d69840ab92

1.43.1
51.4 KiB1561 linesraw
1#include <stdio.h>
2
3#include <vector>
4#include <set>
5
6#include "rustllvm.h"
7
8#include "llvm/Analysis/TargetLibraryInfo.h"
9#include "llvm/Analysis/TargetTransformInfo.h"
10#include "llvm/CodeGen/TargetSubtargetInfo.h"
11#include "llvm/InitializePasses.h"
12#include "llvm/IR/AutoUpgrade.h"
13#include "llvm/IR/AssemblyAnnotationWriter.h"
14#include "llvm/IR/IntrinsicInst.h"
15#include "llvm/IR/Verifier.h"
16#include "llvm/Passes/PassBuilder.h"
17#if LLVM_VERSION_GE(9, 0)
18#include "llvm/Passes/StandardInstrumentations.h"
19#endif
20#include "llvm/Support/CBindingWrapping.h"
21#include "llvm/Support/FileSystem.h"
22#include "llvm/Support/Host.h"
23#include "llvm/Target/TargetMachine.h"
24#include "llvm/Transforms/IPO/PassManagerBuilder.h"
25#include "llvm/Transforms/IPO/AlwaysInliner.h"
26#include "llvm/Transforms/IPO/FunctionImport.h"
27#include "llvm/Transforms/Utils/FunctionImportUtils.h"
28#include "llvm/LTO/LTO.h"
29#include "llvm-c/Transforms/PassManagerBuilder.h"
30
31#include "llvm/Transforms/Instrumentation.h"
32#if LLVM_VERSION_GE(9, 0)
33#include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
34#include "llvm/Support/TimeProfiler.h"
35#endif
36#if LLVM_VERSION_GE(8, 0)
37#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
38#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
39#endif
40#if LLVM_VERSION_GE(9, 0)
41#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
42#endif
43#include "llvm/Transforms/Utils/NameAnonGlobals.h"
44
45using namespace llvm;
46
47typedef struct LLVMOpaquePass *LLVMPassRef;
48typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
49
50DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
51DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
52DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder,
53                                   LLVMPassManagerBuilderRef)
54
55extern "C" void LLVMInitializePasses() {
56  PassRegistry &Registry = *PassRegistry::getPassRegistry();
57  initializeCore(Registry);
58  initializeCodeGen(Registry);
59  initializeScalarOpts(Registry);
60  initializeVectorization(Registry);
61  initializeIPO(Registry);
62  initializeAnalysis(Registry);
63  initializeTransformUtils(Registry);
64  initializeInstCombine(Registry);
65  initializeInstrumentation(Registry);
66  initializeTarget(Registry);
67}
68
69extern "C" void LLVMTimeTraceProfilerInitialize() {
70#if LLVM_VERSION_GE(9, 0)
71  timeTraceProfilerInitialize();
72#endif
73}
74
75extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
76#if LLVM_VERSION_GE(9, 0)
77  StringRef FN(FileName);
78  std::error_code EC;
79  raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
80
81  timeTraceProfilerWrite(OS);
82  timeTraceProfilerCleanup();
83#endif
84}
85
86enum class LLVMRustPassKind {
87  Other,
88  Function,
89  Module,
90};
91
92static LLVMRustPassKind toRust(PassKind Kind) {
93  switch (Kind) {
94  case PT_Function:
95    return LLVMRustPassKind::Function;
96  case PT_Module:
97    return LLVMRustPassKind::Module;
98  default:
99    return LLVMRustPassKind::Other;
100  }
101}
102
103extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
104  StringRef SR(PassName);
105  PassRegistry *PR = PassRegistry::getPassRegistry();
106
107  const PassInfo *PI = PR->getPassInfo(SR);
108  if (PI) {
109    return wrap(PI->createPass());
110  }
111  return nullptr;
112}
113
114extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
115  const bool CompileKernel = false;
116  const bool UseAfterScope = true;
117
118  return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope));
119}
120
121extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
122  const bool CompileKernel = false;
123
124#if LLVM_VERSION_GE(9, 0)
125  return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover));
126#else
127  return wrap(createAddressSanitizerModulePass(CompileKernel, Recover));
128#endif
129}
130
131extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) {
132#if LLVM_VERSION_GE(9, 0)
133  const bool CompileKernel = false;
134
135  return wrap(createMemorySanitizerLegacyPassPass(
136      MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel}));
137#elif LLVM_VERSION_GE(8, 0)
138  return wrap(createMemorySanitizerLegacyPassPass(TrackOrigins, Recover));
139#else
140  return wrap(createMemorySanitizerPass(TrackOrigins, Recover));
141#endif
142}
143
144extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() {
145#if LLVM_VERSION_GE(8, 0)
146  return wrap(createThreadSanitizerLegacyPassPass());
147#else
148  return wrap(createThreadSanitizerPass());
149#endif
150}
151
152extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) {
153  assert(RustPass);
154  Pass *Pass = unwrap(RustPass);
155  return toRust(Pass->getPassKind());
156}
157
158extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
159  assert(RustPass);
160  Pass *Pass = unwrap(RustPass);
161  PassManagerBase *PMB = unwrap(PMR);
162  PMB->add(Pass);
163}
164
165extern "C"
166void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
167  LLVMPassManagerBuilderRef PMBR,
168  LLVMPassManagerRef PMR
169) {
170  unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
171}
172
173extern "C"
174void LLVMRustAddLastExtensionPasses(
175    LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
176  auto AddExtensionPasses = [Passes, NumPasses](
177      const PassManagerBuilder &Builder, PassManagerBase &PM) {
178    for (size_t I = 0; I < NumPasses; I++) {
179      PM.add(unwrap(Passes[I]));
180    }
181  };
182  // Add the passes to both of the pre-finalization extension points,
183  // so they are run for optimized and non-optimized builds.
184  unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
185                             AddExtensionPasses);
186  unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
187                             AddExtensionPasses);
188}
189
190#ifdef LLVM_COMPONENT_X86
191#define SUBTARGET_X86 SUBTARGET(X86)
192#else
193#define SUBTARGET_X86
194#endif
195
196#ifdef LLVM_COMPONENT_ARM
197#define SUBTARGET_ARM SUBTARGET(ARM)
198#else
199#define SUBTARGET_ARM
200#endif
201
202#ifdef LLVM_COMPONENT_AARCH64
203#define SUBTARGET_AARCH64 SUBTARGET(AArch64)
204#else
205#define SUBTARGET_AARCH64
206#endif
207
208#ifdef LLVM_COMPONENT_MIPS
209#define SUBTARGET_MIPS SUBTARGET(Mips)
210#else
211#define SUBTARGET_MIPS
212#endif
213
214#ifdef LLVM_COMPONENT_POWERPC
215#define SUBTARGET_PPC SUBTARGET(PPC)
216#else
217#define SUBTARGET_PPC
218#endif
219
220#ifdef LLVM_COMPONENT_SYSTEMZ
221#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
222#else
223#define SUBTARGET_SYSTEMZ
224#endif
225
226#ifdef LLVM_COMPONENT_MSP430
227#define SUBTARGET_MSP430 SUBTARGET(MSP430)
228#else
229#define SUBTARGET_MSP430
230#endif
231
232#ifdef LLVM_COMPONENT_RISCV
233#define SUBTARGET_RISCV SUBTARGET(RISCV)
234#else
235#define SUBTARGET_RISCV
236#endif
237
238#ifdef LLVM_COMPONENT_SPARC
239#define SUBTARGET_SPARC SUBTARGET(Sparc)
240#else
241#define SUBTARGET_SPARC
242#endif
243
244#ifdef LLVM_COMPONENT_HEXAGON
245#define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
246#else
247#define SUBTARGET_HEXAGON
248#endif
249
250#define GEN_SUBTARGETS                                                         \
251  SUBTARGET_X86                                                                \
252  SUBTARGET_ARM                                                                \
253  SUBTARGET_AARCH64                                                            \
254  SUBTARGET_MIPS                                                               \
255  SUBTARGET_PPC                                                                \
256  SUBTARGET_SYSTEMZ                                                            \
257  SUBTARGET_MSP430                                                             \
258  SUBTARGET_SPARC                                                              \
259  SUBTARGET_HEXAGON                                                            \
260  SUBTARGET_RISCV                                                              \
261
262#define SUBTARGET(x)                                                           \
263  namespace llvm {                                                             \
264  extern const SubtargetFeatureKV x##FeatureKV[];                              \
265  extern const SubtargetFeatureKV x##SubTypeKV[];                              \
266  }
267
268GEN_SUBTARGETS
269#undef SUBTARGET
270
271extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
272                                   const char *Feature) {
273  TargetMachine *Target = unwrap(TM);
274  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
275  return MCInfo->checkFeatures(std::string("+") + Feature);
276}
277
278enum class LLVMRustCodeModel {
279  Other,
280  Small,
281  Kernel,
282  Medium,
283  Large,
284  None,
285};
286
287static CodeModel::Model fromRust(LLVMRustCodeModel Model) {
288  switch (Model) {
289  case LLVMRustCodeModel::Small:
290    return CodeModel::Small;
291  case LLVMRustCodeModel::Kernel:
292    return CodeModel::Kernel;
293  case LLVMRustCodeModel::Medium:
294    return CodeModel::Medium;
295  case LLVMRustCodeModel::Large:
296    return CodeModel::Large;
297  default:
298    report_fatal_error("Bad CodeModel.");
299  }
300}
301
302enum class LLVMRustCodeGenOptLevel {
303  Other,
304  None,
305  Less,
306  Default,
307  Aggressive,
308};
309
310static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
311  switch (Level) {
312  case LLVMRustCodeGenOptLevel::None:
313    return CodeGenOpt::None;
314  case LLVMRustCodeGenOptLevel::Less:
315    return CodeGenOpt::Less;
316  case LLVMRustCodeGenOptLevel::Default:
317    return CodeGenOpt::Default;
318  case LLVMRustCodeGenOptLevel::Aggressive:
319    return CodeGenOpt::Aggressive;
320  default:
321    report_fatal_error("Bad CodeGenOptLevel.");
322  }
323}
324
325enum class LLVMRustPassBuilderOptLevel {
326  O0,
327  O1,
328  O2,
329  O3,
330  Os,
331  Oz,
332};
333
334static PassBuilder::OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
335  switch (Level) {
336  case LLVMRustPassBuilderOptLevel::O0:
337    return PassBuilder::O0;
338  case LLVMRustPassBuilderOptLevel::O1:
339    return PassBuilder::O1;
340  case LLVMRustPassBuilderOptLevel::O2:
341    return PassBuilder::O2;
342  case LLVMRustPassBuilderOptLevel::O3:
343    return PassBuilder::O3;
344  case LLVMRustPassBuilderOptLevel::Os:
345    return PassBuilder::Os;
346  case LLVMRustPassBuilderOptLevel::Oz:
347    return PassBuilder::Oz;
348  default:
349    report_fatal_error("Bad PassBuilderOptLevel.");
350  }
351}
352
353enum class LLVMRustRelocMode {
354  Default,
355  Static,
356  PIC,
357  DynamicNoPic,
358  ROPI,
359  RWPI,
360  ROPIRWPI,
361};
362
363static Optional<Reloc::Model> fromRust(LLVMRustRelocMode RustReloc) {
364  switch (RustReloc) {
365  case LLVMRustRelocMode::Default:
366    return None;
367  case LLVMRustRelocMode::Static:
368    return Reloc::Static;
369  case LLVMRustRelocMode::PIC:
370    return Reloc::PIC_;
371  case LLVMRustRelocMode::DynamicNoPic:
372    return Reloc::DynamicNoPIC;
373  case LLVMRustRelocMode::ROPI:
374    return Reloc::ROPI;
375  case LLVMRustRelocMode::RWPI:
376    return Reloc::RWPI;
377  case LLVMRustRelocMode::ROPIRWPI:
378    return Reloc::ROPI_RWPI;
379  }
380  report_fatal_error("Bad RelocModel.");
381}
382
383#ifdef LLVM_RUSTLLVM
384/// getLongestEntryLength - Return the length of the longest entry in the table.
385template<typename KV>
386static size_t getLongestEntryLength(ArrayRef<KV> Table) {
387  size_t MaxLen = 0;
388  for (auto &I : Table)
389    MaxLen = std::max(MaxLen, std::strlen(I.Key));
390  return MaxLen;
391}
392
393extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
394  const TargetMachine *Target = unwrap(TM);
395  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
396  const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
397  const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
398  const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
399  unsigned MaxCPULen = getLongestEntryLength(CPUTable);
400
401  printf("Available CPUs for this target:\n");
402  if (HostArch == TargetArch) {
403    const StringRef HostCPU = sys::getHostCPUName();
404    printf("    %-*s - Select the CPU of the current host (currently %.*s).\n",
405      MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
406  }
407  for (auto &CPU : CPUTable)
408    printf("    %-*s\n", MaxCPULen, CPU.Key);
409  printf("\n");
410}
411
412extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) {
413  const TargetMachine *Target = unwrap(TM);
414  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
415  const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
416  unsigned MaxFeatLen = getLongestEntryLength(FeatTable);
417
418  printf("Available features for this target:\n");
419  for (auto &Feature : FeatTable)
420    printf("    %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc);
421  printf("\n");
422
423  printf("Use +feature to enable a feature, or -feature to disable it.\n"
424         "For example, rustc -C -target-cpu=mycpu -C "
425         "target-feature=+feature1,-feature2\n\n");
426}
427
428#else
429
430extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
431  printf("Target CPU help is not supported by this LLVM version.\n\n");
432}
433
434extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) {
435  printf("Target features help is not supported by this LLVM version.\n\n");
436}
437#endif
438
439extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
440  StringRef Name = sys::getHostCPUName();
441  *len = Name.size();
442  return Name.data();
443}
444
445extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
446    const char *TripleStr, const char *CPU, const char *Feature,
447    const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocMode RustReloc,
448    LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
449    bool PositionIndependentExecutable, bool FunctionSections,
450    bool DataSections,
451    bool TrapUnreachable,
452    bool Singlethread,
453    bool AsmComments,
454    bool EmitStackSizeSection,
455    bool RelaxELFRelocations) {
456
457  auto OptLevel = fromRust(RustOptLevel);
458  auto RM = fromRust(RustReloc);
459
460  std::string Error;
461  Triple Trip(Triple::normalize(TripleStr));
462  const llvm::Target *TheTarget =
463      TargetRegistry::lookupTarget(Trip.getTriple(), Error);
464  if (TheTarget == nullptr) {
465    LLVMRustSetLastError(Error.c_str());
466    return nullptr;
467  }
468
469  TargetOptions Options;
470
471  Options.FloatABIType = FloatABI::Default;
472  if (UseSoftFloat) {
473    Options.FloatABIType = FloatABI::Soft;
474  }
475  Options.DataSections = DataSections;
476  Options.FunctionSections = FunctionSections;
477  Options.MCOptions.AsmVerbose = AsmComments;
478  Options.MCOptions.PreserveAsmComments = AsmComments;
479  Options.MCOptions.ABIName = ABIStr;
480  Options.RelaxELFRelocations = RelaxELFRelocations;
481
482  if (TrapUnreachable) {
483    // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
484    // This limits the extent of possible undefined behavior in some cases, as
485    // it prevents control flow from "falling through" into whatever code
486    // happens to be laid out next in memory.
487    Options.TrapUnreachable = true;
488  }
489
490  if (Singlethread) {
491    Options.ThreadModel = ThreadModel::Single;
492  }
493
494  Options.EmitStackSizeSection = EmitStackSizeSection;
495
496  Optional<CodeModel::Model> CM;
497  if (RustCM != LLVMRustCodeModel::None)
498    CM = fromRust(RustCM);
499  TargetMachine *TM = TheTarget->createTargetMachine(
500      Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
501  return wrap(TM);
502}
503
504extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
505  delete unwrap(TM);
506}
507
508extern "C" void LLVMRustConfigurePassManagerBuilder(
509    LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
510    bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
511    const char* PGOGenPath, const char* PGOUsePath) {
512  unwrap(PMBR)->MergeFunctions = MergeFunctions;
513  unwrap(PMBR)->SLPVectorize = SLPVectorize;
514  unwrap(PMBR)->OptLevel = fromRust(OptLevel);
515  unwrap(PMBR)->LoopVectorize = LoopVectorize;
516  unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
517
518  if (PGOGenPath) {
519    assert(!PGOUsePath);
520    unwrap(PMBR)->EnablePGOInstrGen = true;
521    unwrap(PMBR)->PGOInstrGen = PGOGenPath;
522  }
523  if (PGOUsePath) {
524    assert(!PGOGenPath);
525    unwrap(PMBR)->PGOInstrUse = PGOUsePath;
526  }
527}
528
529// Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
530// field of a PassManagerBuilder, we expose our own method of doing so.
531extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
532                                              LLVMModuleRef M,
533                                              bool DisableSimplifyLibCalls) {
534  Triple TargetTriple(unwrap(M)->getTargetTriple());
535  TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
536  if (DisableSimplifyLibCalls)
537    TLI->disableAllFunctions();
538  unwrap(PMBR)->LibraryInfo = TLI;
539}
540
541// Unfortunately, the LLVM C API doesn't provide a way to create the
542// TargetLibraryInfo pass, so we use this method to do so.
543extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
544                                       bool DisableSimplifyLibCalls) {
545  Triple TargetTriple(unwrap(M)->getTargetTriple());
546  TargetLibraryInfoImpl TLII(TargetTriple);
547  if (DisableSimplifyLibCalls)
548    TLII.disableAllFunctions();
549  unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
550}
551
552// Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
553// all the functions in a module, so we do that manually here. You'll find
554// similar code in clang's BackendUtil.cpp file.
555extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
556                                               LLVMModuleRef M) {
557  llvm::legacy::FunctionPassManager *P =
558      unwrap<llvm::legacy::FunctionPassManager>(PMR);
559  P->doInitialization();
560
561  // Upgrade all calls to old intrinsics first.
562  for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
563    UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
564
565  for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
566       ++I)
567    if (!I->isDeclaration())
568      P->run(*I);
569
570  P->doFinalization();
571}
572
573extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
574  // Initializing the command-line options more than once is not allowed. So,
575  // check if they've already been initialized.  (This could happen if we're
576  // being called from rustpkg, for example). If the arguments change, then
577  // that's just kinda unfortunate.
578  static bool Initialized = false;
579  if (Initialized)
580    return;
581  Initialized = true;
582  cl::ParseCommandLineOptions(Argc, Argv);
583}
584
585enum class LLVMRustFileType {
586  Other,
587  AssemblyFile,
588  ObjectFile,
589};
590
591#if LLVM_VERSION_GE(10, 0)
592static CodeGenFileType fromRust(LLVMRustFileType Type) {
593  switch (Type) {
594  case LLVMRustFileType::AssemblyFile:
595    return CGFT_AssemblyFile;
596  case LLVMRustFileType::ObjectFile:
597    return CGFT_ObjectFile;
598  default:
599    report_fatal_error("Bad FileType.");
600  }
601}
602#else
603static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) {
604  switch (Type) {
605  case LLVMRustFileType::AssemblyFile:
606    return TargetMachine::CGFT_AssemblyFile;
607  case LLVMRustFileType::ObjectFile:
608    return TargetMachine::CGFT_ObjectFile;
609  default:
610    report_fatal_error("Bad FileType.");
611  }
612}
613#endif
614
615extern "C" LLVMRustResult
616LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
617                        LLVMModuleRef M, const char *Path,
618                        LLVMRustFileType RustFileType) {
619  llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
620  auto FileType = fromRust(RustFileType);
621
622  std::string ErrorInfo;
623  std::error_code EC;
624  raw_fd_ostream OS(Path, EC, sys::fs::F_None);
625  if (EC)
626    ErrorInfo = EC.message();
627  if (ErrorInfo != "") {
628    LLVMRustSetLastError(ErrorInfo.c_str());
629    return LLVMRustResult::Failure;
630  }
631
632  buffer_ostream BOS(OS);
633  unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
634  PM->run(*unwrap(M));
635
636  // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
637  // stream (OS), so the only real safe place to delete this is here? Don't we
638  // wish this was written in Rust?
639  LLVMDisposePassManager(PMR);
640  return LLVMRustResult::Success;
641}
642
643extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
644                                                      const char*,      // pass name
645                                                      const char*);     // IR name
646extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
647
648#if LLVM_VERSION_GE(9, 0)
649
650std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
651  if (any_isa<const Module *>(WrappedIr))
652    return any_cast<const Module *>(WrappedIr)->getName().str();
653  if (any_isa<const Function *>(WrappedIr))
654    return any_cast<const Function *>(WrappedIr)->getName().str();
655  if (any_isa<const Loop *>(WrappedIr))
656    return any_cast<const Loop *>(WrappedIr)->getName().str();
657  if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
658    return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
659  return "<UNKNOWN>";
660}
661
662
663void LLVMSelfProfileInitializeCallbacks(
664    PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
665    LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
666    LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
667  PIC.registerBeforePassCallback([LlvmSelfProfiler, BeforePassCallback](
668                                     StringRef Pass, llvm::Any Ir) {
669    std::string PassName = Pass.str();
670    std::string IrName = LLVMRustwrappedIrGetName(Ir);
671    BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
672    return true;
673  });
674
675  PIC.registerAfterPassCallback(
676      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
677        AfterPassCallback(LlvmSelfProfiler);
678      });
679
680  PIC.registerAfterPassInvalidatedCallback(
681      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass) {
682        AfterPassCallback(LlvmSelfProfiler);
683      });
684
685  PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
686                                         StringRef Pass, llvm::Any Ir) {
687    std::string PassName = Pass.str();
688    std::string IrName = LLVMRustwrappedIrGetName(Ir);
689    BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
690  });
691
692  PIC.registerAfterAnalysisCallback(
693      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
694        AfterPassCallback(LlvmSelfProfiler);
695      });
696}
697#endif
698
699enum class LLVMRustOptStage {
700  PreLinkNoLTO,
701  PreLinkThinLTO,
702  PreLinkFatLTO,
703  ThinLTO,
704  FatLTO,
705};
706
707struct LLVMRustSanitizerOptions {
708  bool SanitizeMemory;
709  bool SanitizeThread;
710  bool SanitizeAddress;
711  bool SanitizeRecover;
712  int SanitizeMemoryTrackOrigins;
713};
714
715extern "C" void
716LLVMRustOptimizeWithNewPassManager(
717    LLVMModuleRef ModuleRef,
718    LLVMTargetMachineRef TMRef,
719    LLVMRustPassBuilderOptLevel OptLevelRust,
720    LLVMRustOptStage OptStage,
721    bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
722    bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
723    bool DisableSimplifyLibCalls,
724    LLVMRustSanitizerOptions *SanitizerOptions,
725    const char *PGOGenPath, const char *PGOUsePath,
726    void* LlvmSelfProfiler,
727    LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
728    LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
729#if LLVM_VERSION_GE(9, 0)
730  Module *TheModule = unwrap(ModuleRef);
731  TargetMachine *TM = unwrap(TMRef);
732  PassBuilder::OptimizationLevel OptLevel = fromRust(OptLevelRust);
733
734  // FIXME: MergeFunctions is not supported by NewPM yet.
735  (void) MergeFunctions;
736
737  PipelineTuningOptions PTO;
738  PTO.LoopUnrolling = UnrollLoops;
739  PTO.LoopInterleaving = UnrollLoops;
740  PTO.LoopVectorization = LoopVectorize;
741  PTO.SLPVectorization = SLPVectorize;
742
743  PassInstrumentationCallbacks PIC;
744  StandardInstrumentations SI;
745  SI.registerCallbacks(PIC);
746
747  if (LlvmSelfProfiler){
748    LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
749  }
750
751  Optional<PGOOptions> PGOOpt;
752  if (PGOGenPath) {
753    assert(!PGOUsePath);
754    PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr);
755  } else if (PGOUsePath) {
756    assert(!PGOGenPath);
757    PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse);
758  }
759
760  PassBuilder PB(TM, PTO, PGOOpt, &PIC);
761
762  // FIXME: We may want to expose this as an option.
763  bool DebugPassManager = false;
764  LoopAnalysisManager LAM(DebugPassManager);
765  FunctionAnalysisManager FAM(DebugPassManager);
766  CGSCCAnalysisManager CGAM(DebugPassManager);
767  ModuleAnalysisManager MAM(DebugPassManager);
768
769  FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
770
771  Triple TargetTriple(TheModule->getTargetTriple());
772  std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
773  if (DisableSimplifyLibCalls)
774    TLII->disableAllFunctions();
775  FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
776
777  PB.registerModuleAnalyses(MAM);
778  PB.registerCGSCCAnalyses(CGAM);
779  PB.registerFunctionAnalyses(FAM);
780  PB.registerLoopAnalyses(LAM);
781  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
782
783  // We manually collect pipeline callbacks so we can apply them at O0, where the
784  // PassBuilder does not create a pipeline.
785  std::vector<std::function<void(ModulePassManager &)>> PipelineStartEPCallbacks;
786  std::vector<std::function<void(FunctionPassManager &, PassBuilder::OptimizationLevel)>>
787      OptimizerLastEPCallbacks;
788
789  if (VerifyIR) {
790    PipelineStartEPCallbacks.push_back([VerifyIR](ModulePassManager &MPM) {
791        MPM.addPass(VerifierPass());
792    });
793  }
794
795  if (SanitizerOptions) {
796    if (SanitizerOptions->SanitizeMemory) {
797      MemorySanitizerOptions Options(
798          SanitizerOptions->SanitizeMemoryTrackOrigins,
799          SanitizerOptions->SanitizeRecover,
800          /*CompileKernel=*/false);
801#if LLVM_VERSION_GE(10, 0)
802      PipelineStartEPCallbacks.push_back([Options](ModulePassManager &MPM) {
803        MPM.addPass(MemorySanitizerPass(Options));
804      });
805#endif
806      OptimizerLastEPCallbacks.push_back(
807        [Options](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
808          FPM.addPass(MemorySanitizerPass(Options));
809        }
810      );
811    }
812
813    if (SanitizerOptions->SanitizeThread) {
814#if LLVM_VERSION_GE(10, 0)
815      PipelineStartEPCallbacks.push_back([](ModulePassManager &MPM) {
816        MPM.addPass(ThreadSanitizerPass());
817      });
818#endif
819      OptimizerLastEPCallbacks.push_back(
820        [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
821          FPM.addPass(ThreadSanitizerPass());
822        }
823      );
824    }
825
826    if (SanitizerOptions->SanitizeAddress) {
827      PipelineStartEPCallbacks.push_back([&](ModulePassManager &MPM) {
828        MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
829      });
830      OptimizerLastEPCallbacks.push_back(
831        [SanitizerOptions](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
832          FPM.addPass(AddressSanitizerPass(
833              /*CompileKernel=*/false, SanitizerOptions->SanitizeRecover,
834              /*UseAfterScope=*/true));
835        }
836      );
837      PipelineStartEPCallbacks.push_back(
838        [SanitizerOptions](ModulePassManager &MPM) {
839          MPM.addPass(ModuleAddressSanitizerPass(
840              /*CompileKernel=*/false, SanitizerOptions->SanitizeRecover));
841        }
842      );
843    }
844  }
845
846  ModulePassManager MPM(DebugPassManager);
847  if (!NoPrepopulatePasses) {
848    if (OptLevel == PassBuilder::O0) {
849      for (const auto &C : PipelineStartEPCallbacks)
850        C(MPM);
851
852      if (!OptimizerLastEPCallbacks.empty()) {
853        FunctionPassManager FPM(DebugPassManager);
854        for (const auto &C : OptimizerLastEPCallbacks)
855          C(FPM, OptLevel);
856        MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
857      }
858
859      MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/false));
860
861#if LLVM_VERSION_GE(10, 0)
862      if (PGOOpt) {
863        PB.addPGOInstrPassesForO0(
864            MPM, DebugPassManager, PGOOpt->Action == PGOOptions::IRInstr,
865            /*IsCS=*/false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
866      }
867#endif
868    } else {
869      for (const auto &C : PipelineStartEPCallbacks)
870        PB.registerPipelineStartEPCallback(C);
871      if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
872        for (const auto &C : OptimizerLastEPCallbacks)
873          PB.registerOptimizerLastEPCallback(C);
874      }
875
876      switch (OptStage) {
877      case LLVMRustOptStage::PreLinkNoLTO:
878        MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
879        break;
880      case LLVMRustOptStage::PreLinkThinLTO:
881        MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
882        if (!OptimizerLastEPCallbacks.empty()) {
883          FunctionPassManager FPM(DebugPassManager);
884          for (const auto &C : OptimizerLastEPCallbacks)
885            C(FPM, OptLevel);
886          MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
887        }
888        break;
889      case LLVMRustOptStage::PreLinkFatLTO:
890        MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
891        break;
892      case LLVMRustOptStage::ThinLTO:
893        // FIXME: Does it make sense to pass the ModuleSummaryIndex?
894        // It only seems to be needed for C++ specific optimizations.
895        MPM = PB.buildThinLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
896        break;
897      case LLVMRustOptStage::FatLTO:
898        MPM = PB.buildLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
899        break;
900      }
901    }
902  }
903
904  if (UseThinLTOBuffers) {
905    MPM.addPass(CanonicalizeAliasesPass());
906    MPM.addPass(NameAnonGlobalPass());
907  }
908
909  // Upgrade all calls to old intrinsics first.
910  for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
911    UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
912
913  MPM.run(*TheModule, MAM);
914#else
915  // The new pass manager has been available for a long time,
916  // but we don't bother supporting it on old LLVM versions.
917  report_fatal_error("New pass manager only supported since LLVM 9");
918#endif
919}
920
921// Callback to demangle function name
922// Parameters:
923// * name to be demangled
924// * name len
925// * output buffer
926// * output buffer len
927// Returns len of demangled string, or 0 if demangle failed.
928typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
929
930
931namespace {
932
933class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
934  DemangleFn Demangle;
935  std::vector<char> Buf;
936
937public:
938  RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
939
940  // Return empty string if demangle failed
941  // or if name does not need to be demangled
942  StringRef CallDemangle(StringRef name) {
943    if (!Demangle) {
944      return StringRef();
945    }
946
947    if (Buf.size() < name.size() * 2) {
948      // Semangled name usually shorter than mangled,
949      // but allocate twice as much memory just in case
950      Buf.resize(name.size() * 2);
951    }
952
953    auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
954    if (!R) {
955      // Demangle failed.
956      return StringRef();
957    }
958
959    auto Demangled = StringRef(Buf.data(), R);
960    if (Demangled == name) {
961      // Do not print anything if demangled name is equal to mangled.
962      return StringRef();
963    }
964
965    return Demangled;
966  }
967
968  void emitFunctionAnnot(const Function *F,
969                         formatted_raw_ostream &OS) override {
970    StringRef Demangled = CallDemangle(F->getName());
971    if (Demangled.empty()) {
972        return;
973    }
974
975    OS << "; " << Demangled << "\n";
976  }
977
978  void emitInstructionAnnot(const Instruction *I,
979                            formatted_raw_ostream &OS) override {
980    const char *Name;
981    const Value *Value;
982    if (const CallInst *CI = dyn_cast<CallInst>(I)) {
983      Name = "call";
984      Value = CI->getCalledValue();
985    } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
986      Name = "invoke";
987      Value = II->getCalledValue();
988    } else {
989      // Could demangle more operations, e. g.
990      // `store %place, @function`.
991      return;
992    }
993
994    if (!Value->hasName()) {
995      return;
996    }
997
998    StringRef Demangled = CallDemangle(Value->getName());
999    if (Demangled.empty()) {
1000      return;
1001    }
1002
1003    OS << "; " << Name << " " << Demangled << "\n";
1004  }
1005};
1006
1007} // namespace
1008
1009extern "C" LLVMRustResult
1010LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
1011  std::string ErrorInfo;
1012  std::error_code EC;
1013  raw_fd_ostream OS(Path, EC, sys::fs::F_None);
1014  if (EC)
1015    ErrorInfo = EC.message();
1016  if (ErrorInfo != "") {
1017    LLVMRustSetLastError(ErrorInfo.c_str());
1018    return LLVMRustResult::Failure;
1019  }
1020
1021  RustAssemblyAnnotationWriter AAW(Demangle);
1022  formatted_raw_ostream FOS(OS);
1023  unwrap(M)->print(FOS, &AAW);
1024
1025  return LLVMRustResult::Success;
1026}
1027
1028extern "C" void LLVMRustPrintPasses() {
1029  LLVMInitializePasses();
1030  struct MyListener : PassRegistrationListener {
1031    void passEnumerate(const PassInfo *Info) {
1032      StringRef PassArg = Info->getPassArgument();
1033      StringRef PassName = Info->getPassName();
1034      if (!PassArg.empty()) {
1035        // These unsigned->signed casts could theoretically overflow, but
1036        // realistically never will (and even if, the result is implementation
1037        // defined rather plain UB).
1038        printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
1039               (int)PassName.size(), PassName.data());
1040      }
1041    }
1042  } Listener;
1043
1044  PassRegistry *PR = PassRegistry::getPassRegistry();
1045  PR->enumerateWith(&Listener);
1046}
1047
1048extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
1049                                            bool AddLifetimes) {
1050  unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
1051}
1052
1053extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1054                                           size_t Len) {
1055  llvm::legacy::PassManager passes;
1056
1057  auto PreserveFunctions = [=](const GlobalValue &GV) {
1058    for (size_t I = 0; I < Len; I++) {
1059      if (GV.getName() == Symbols[I]) {
1060        return true;
1061      }
1062    }
1063    return false;
1064  };
1065
1066  passes.add(llvm::createInternalizePass(PreserveFunctions));
1067
1068  passes.run(*unwrap(M));
1069}
1070
1071extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) {
1072  for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E;
1073       ++GV) {
1074    GV->setDoesNotThrow();
1075    Function *F = dyn_cast<Function>(GV);
1076    if (F == nullptr)
1077      continue;
1078
1079    for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
1080      for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) {
1081        if (isa<InvokeInst>(I)) {
1082          InvokeInst *CI = cast<InvokeInst>(I);
1083          CI->setDoesNotThrow();
1084        }
1085      }
1086    }
1087  }
1088}
1089
1090extern "C" void
1091LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1092                                       LLVMTargetMachineRef TMR) {
1093  TargetMachine *Target = unwrap(TMR);
1094  unwrap(Module)->setDataLayout(Target->createDataLayout());
1095}
1096
1097extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1098  unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1099}
1100
1101extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1102  unwrap(M)->setPIELevel(PIELevel::Level::Large);
1103}
1104
1105// Here you'll find an implementation of ThinLTO as used by the Rust compiler
1106// right now. This ThinLTO support is only enabled on "recent ish" versions of
1107// LLVM, and otherwise it's just blanket rejected from other compilers.
1108//
1109// Most of this implementation is straight copied from LLVM. At the time of
1110// this writing it wasn't *quite* suitable to reuse more code from upstream
1111// for our purposes, but we should strive to upstream this support once it's
1112// ready to go! I figure we may want a bit of testing locally first before
1113// sending this upstream to LLVM. I hear though they're quite eager to receive
1114// feedback like this!
1115//
1116// If you're reading this code and wondering "what in the world" or you're
1117// working "good lord by LLVM upgrade is *still* failing due to these bindings"
1118// then fear not! (ok maybe fear a little). All code here is mostly based
1119// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1120//
1121// You'll find that the general layout here roughly corresponds to the `run`
1122// method in that file as well as `ProcessThinLTOModule`. Functions are
1123// specifically commented below as well, but if you're updating this code
1124// or otherwise trying to understand it, the LLVM source will be useful in
1125// interpreting the mysteries within.
1126//
1127// Otherwise I'll apologize in advance, it probably requires a relatively
1128// significant investment on your part to "truly understand" what's going on
1129// here. Not saying I do myself, but it took me awhile staring at LLVM's source
1130// and various online resources about ThinLTO to make heads or tails of all
1131// this.
1132
1133// This is a shared data structure which *must* be threadsafe to share
1134// read-only amongst threads. This also corresponds basically to the arguments
1135// of the `ProcessThinLTOModule` function in the LLVM source.
1136struct LLVMRustThinLTOData {
1137  // The combined index that is the global analysis over all modules we're
1138  // performing ThinLTO for. This is mostly managed by LLVM.
1139  ModuleSummaryIndex Index;
1140
1141  // All modules we may look at, stored as in-memory serialized versions. This
1142  // is later used when inlining to ensure we can extract any module to inline
1143  // from.
1144  StringMap<MemoryBufferRef> ModuleMap;
1145
1146  // A set that we manage of everything we *don't* want internalized. Note that
1147  // this includes all transitive references right now as well, but it may not
1148  // always!
1149  DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1150
1151  // Not 100% sure what these are, but they impact what's internalized and
1152  // what's inlined across modules, I believe.
1153  StringMap<FunctionImporter::ImportMapTy> ImportLists;
1154  StringMap<FunctionImporter::ExportSetTy> ExportLists;
1155  StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1156
1157  LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1158};
1159
1160// Just an argument to the `LLVMRustCreateThinLTOData` function below.
1161struct LLVMRustThinLTOModule {
1162  const char *identifier;
1163  const char *data;
1164  size_t len;
1165};
1166
1167// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1168// does.
1169static const GlobalValueSummary *
1170getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1171  auto StrongDefForLinker = llvm::find_if(
1172      GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1173        auto Linkage = Summary->linkage();
1174        return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1175               !GlobalValue::isWeakForLinker(Linkage);
1176      });
1177  if (StrongDefForLinker != GVSummaryList.end())
1178    return StrongDefForLinker->get();
1179
1180  auto FirstDefForLinker = llvm::find_if(
1181      GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1182        auto Linkage = Summary->linkage();
1183        return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1184      });
1185  if (FirstDefForLinker == GVSummaryList.end())
1186    return nullptr;
1187  return FirstDefForLinker->get();
1188}
1189
1190// The main entry point for creating the global ThinLTO analysis. The structure
1191// here is basically the same as before threads are spawned in the `run`
1192// function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1193extern "C" LLVMRustThinLTOData*
1194LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1195                          int num_modules,
1196                          const char **preserved_symbols,
1197                          int num_symbols) {
1198#if LLVM_VERSION_GE(10, 0)
1199  auto Ret = std::make_unique<LLVMRustThinLTOData>();
1200#else
1201  auto Ret = llvm::make_unique<LLVMRustThinLTOData>();
1202#endif
1203
1204  // Load each module's summary and merge it into one combined index
1205  for (int i = 0; i < num_modules; i++) {
1206    auto module = &modules[i];
1207    StringRef buffer(module->data, module->len);
1208    MemoryBufferRef mem_buffer(buffer, module->identifier);
1209
1210    Ret->ModuleMap[module->identifier] = mem_buffer;
1211
1212    if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1213      LLVMRustSetLastError(toString(std::move(Err)).c_str());
1214      return nullptr;
1215    }
1216  }
1217
1218  // Collect for each module the list of function it defines (GUID -> Summary)
1219  Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1220
1221  // Convert the preserved symbols set from string to GUID, this is then needed
1222  // for internalization.
1223  for (int i = 0; i < num_symbols; i++) {
1224    auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1225    Ret->GUIDPreservedSymbols.insert(GUID);
1226  }
1227
1228  // Collect the import/export lists for all modules from the call-graph in the
1229  // combined index
1230  //
1231  // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1232  auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1233    return PrevailingType::Unknown;
1234  };
1235#if LLVM_VERSION_GE(8, 0)
1236  // We don't have a complete picture in our use of ThinLTO, just our immediate
1237  // crate, so we need `ImportEnabled = false` to limit internalization.
1238  // Otherwise, we sometimes lose `static` values -- see #60184.
1239  computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1240                                  deadIsPrevailing, /* ImportEnabled = */ false);
1241#else
1242  computeDeadSymbols(Ret->Index, Ret->GUIDPreservedSymbols, deadIsPrevailing);
1243#endif
1244  ComputeCrossModuleImport(
1245    Ret->Index,
1246    Ret->ModuleToDefinedGVSummaries,
1247    Ret->ImportLists,
1248    Ret->ExportLists
1249  );
1250
1251  // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1252  // impacts the caching.
1253  //
1254  // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1255  // being lifted from `lib/LTO/LTO.cpp` as well
1256  StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1257  DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1258  for (auto &I : Ret->Index) {
1259    if (I.second.SummaryList.size() > 1)
1260      PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1261  }
1262  auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1263    const auto &Prevailing = PrevailingCopy.find(GUID);
1264    if (Prevailing == PrevailingCopy.end())
1265      return true;
1266    return Prevailing->second == S;
1267  };
1268  auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1269                              GlobalValue::GUID GUID,
1270                              GlobalValue::LinkageTypes NewLinkage) {
1271    ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1272  };
1273#if LLVM_VERSION_GE(9, 0)
1274  thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage,
1275                                  Ret->GUIDPreservedSymbols);
1276#elif LLVM_VERSION_GE(8, 0)
1277  thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage);
1278#else
1279  thinLTOResolveWeakForLinkerInIndex(Ret->Index, isPrevailing, recordNewLinkage);
1280#endif
1281
1282  // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1283  // callback below. This callback below will dictate the linkage for all
1284  // summaries in the index, and we basically just only want to ensure that dead
1285  // symbols are internalized. Otherwise everything that's already external
1286  // linkage will stay as external, and internal will stay as internal.
1287  std::set<GlobalValue::GUID> ExportedGUIDs;
1288  for (auto &List : Ret->Index) {
1289    for (auto &GVS: List.second.SummaryList) {
1290      if (GlobalValue::isLocalLinkage(GVS->linkage()))
1291        continue;
1292      auto GUID = GVS->getOriginalName();
1293      if (GVS->flags().Live)
1294        ExportedGUIDs.insert(GUID);
1295    }
1296  }
1297#if LLVM_VERSION_GE(10, 0)
1298  auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1299    const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1300    return (ExportList != Ret->ExportLists.end() &&
1301      ExportList->second.count(VI)) ||
1302      ExportedGUIDs.count(VI.getGUID());
1303  };
1304  thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1305#else
1306  auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
1307    const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1308    return (ExportList != Ret->ExportLists.end() &&
1309      ExportList->second.count(GUID)) ||
1310      ExportedGUIDs.count(GUID);
1311  };
1312  thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported);
1313#endif
1314
1315  return Ret.release();
1316}
1317
1318extern "C" void
1319LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1320  delete Data;
1321}
1322
1323// Below are the various passes that happen *per module* when doing ThinLTO.
1324//
1325// In other words, these are the functions that are all run concurrently
1326// with one another, one per module. The passes here correspond to the analysis
1327// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1328// `ProcessThinLTOModule` function. Here they're split up into separate steps
1329// so rustc can save off the intermediate bytecode between each step.
1330
1331extern "C" bool
1332LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1333  Module &Mod = *unwrap(M);
1334  if (renameModuleForThinLTO(Mod, Data->Index)) {
1335    LLVMRustSetLastError("renameModuleForThinLTO failed");
1336    return false;
1337  }
1338  return true;
1339}
1340
1341extern "C" bool
1342LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1343  Module &Mod = *unwrap(M);
1344  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1345#if LLVM_VERSION_GE(8, 0)
1346  thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1347#else
1348  thinLTOResolveWeakForLinkerModule(Mod, DefinedGlobals);
1349#endif
1350  return true;
1351}
1352
1353extern "C" bool
1354LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1355  Module &Mod = *unwrap(M);
1356  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1357  thinLTOInternalizeModule(Mod, DefinedGlobals);
1358  return true;
1359}
1360
1361extern "C" bool
1362LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1363  Module &Mod = *unwrap(M);
1364
1365  const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1366  auto Loader = [&](StringRef Identifier) {
1367    const auto &Memory = Data->ModuleMap.lookup(Identifier);
1368    auto &Context = Mod.getContext();
1369    auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1370
1371    if (!MOrErr)
1372      return MOrErr;
1373
1374    // The rest of this closure is a workaround for
1375    // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1376    // we accidentally import wasm custom sections into different modules,
1377    // duplicating them by in the final output artifact.
1378    //
1379    // The issue is worked around here by manually removing the
1380    // `wasm.custom_sections` named metadata node from any imported module. This
1381    // we know isn't used by any optimization pass so there's no need for it to
1382    // be imported.
1383    //
1384    // Note that the metadata is currently lazily loaded, so we materialize it
1385    // here before looking up if there's metadata inside. The `FunctionImporter`
1386    // will immediately materialize metadata anyway after an import, so this
1387    // shouldn't be a perf hit.
1388    if (Error Err = (*MOrErr)->materializeMetadata()) {
1389      Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1390      return Ret;
1391    }
1392
1393    auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1394    if (WasmCustomSections)
1395      WasmCustomSections->eraseFromParent();
1396
1397    return MOrErr;
1398  };
1399  FunctionImporter Importer(Data->Index, Loader);
1400  Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1401  if (!Result) {
1402    LLVMRustSetLastError(toString(Result.takeError()).c_str());
1403    return false;
1404  }
1405  return true;
1406}
1407
1408extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload
1409                                                      const char*, // importing module name
1410                                                      const char*); // imported module name
1411
1412// Calls `module_name_callback` for each module import done by ThinLTO.
1413// The callback is provided with regular null-terminated C strings.
1414extern "C" void
1415LLVMRustGetThinLTOModuleImports(const LLVMRustThinLTOData *data,
1416                                LLVMRustModuleNameCallback module_name_callback,
1417                                void* callback_payload) {
1418  for (const auto& importing_module : data->ImportLists) {
1419    const std::string importing_module_id = importing_module.getKey().str();
1420    const auto& imports = importing_module.getValue();
1421    for (const auto& imported_module : imports) {
1422      const std::string imported_module_id = imported_module.getKey().str();
1423      module_name_callback(callback_payload,
1424                           importing_module_id.c_str(),
1425                           imported_module_id.c_str());
1426    }
1427  }
1428}
1429
1430// This struct and various functions are sort of a hack right now, but the
1431// problem is that we've got in-memory LLVM modules after we generate and
1432// optimize all codegen-units for one compilation in rustc. To be compatible
1433// with the LTO support above we need to serialize the modules plus their
1434// ThinLTO summary into memory.
1435//
1436// This structure is basically an owned version of a serialize module, with
1437// a ThinLTO summary attached.
1438struct LLVMRustThinLTOBuffer {
1439  std::string data;
1440};
1441
1442extern "C" LLVMRustThinLTOBuffer*
1443LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
1444#if LLVM_VERSION_GE(10, 0)
1445  auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1446#else
1447  auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>();
1448#endif
1449  {
1450    raw_string_ostream OS(Ret->data);
1451    {
1452      legacy::PassManager PM;
1453      PM.add(createWriteThinLTOBitcodePass(OS));
1454      PM.run(*unwrap(M));
1455    }
1456  }
1457  return Ret.release();
1458}
1459
1460extern "C" void
1461LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1462  delete Buffer;
1463}
1464
1465extern "C" const void*
1466LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1467  return Buffer->data.data();
1468}
1469
1470extern "C" size_t
1471LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1472  return Buffer->data.length();
1473}
1474
1475// This is what we used to parse upstream bitcode for actual ThinLTO
1476// processing.  We'll call this once per module optimized through ThinLTO, and
1477// it'll be called concurrently on many threads.
1478extern "C" LLVMModuleRef
1479LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1480                           const char *data,
1481                           size_t len,
1482                           const char *identifier) {
1483  StringRef Data(data, len);
1484  MemoryBufferRef Buffer(Data, identifier);
1485  unwrap(Context)->enableDebugTypeODRUniquing();
1486  Expected<std::unique_ptr<Module>> SrcOrError =
1487      parseBitcodeFile(Buffer, *unwrap(Context));
1488  if (!SrcOrError) {
1489    LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1490    return nullptr;
1491  }
1492  return wrap(std::move(*SrcOrError).release());
1493}
1494
1495// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1496// the comment in `back/lto.rs` for why this exists.
1497extern "C" void
1498LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1499                                DICompileUnit **A,
1500                                DICompileUnit **B) {
1501  Module *M = unwrap(Mod);
1502  DICompileUnit **Cur = A;
1503  DICompileUnit **Next = B;
1504  for (DICompileUnit *CU : M->debug_compile_units()) {
1505    *Cur = CU;
1506    Cur = Next;
1507    Next = nullptr;
1508    if (Cur == nullptr)
1509      break;
1510  }
1511}
1512
1513// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1514// the comment in `back/lto.rs` for why this exists.
1515extern "C" void
1516LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1517  Module *M = unwrap(Mod);
1518
1519  // If the original source module didn't have a `DICompileUnit` then try to
1520  // merge all the existing compile units. If there aren't actually any though
1521  // then there's not much for us to do so return.
1522  if (Unit == nullptr) {
1523    for (DICompileUnit *CU : M->debug_compile_units()) {
1524      Unit = CU;
1525      break;
1526    }
1527    if (Unit == nullptr)
1528      return;
1529  }
1530
1531  // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1532  // process it recursively. Note that we specifically iterate over instructions
1533  // to ensure we feed everything into it.
1534  DebugInfoFinder Finder;
1535  Finder.processModule(*M);
1536  for (Function &F : M->functions()) {
1537    for (auto &FI : F) {
1538      for (Instruction &BI : FI) {
1539        if (auto Loc = BI.getDebugLoc())
1540          Finder.processLocation(*M, Loc);
1541        if (auto DVI = dyn_cast<DbgValueInst>(&BI))
1542          Finder.processValue(*M, DVI);
1543        if (auto DDI = dyn_cast<DbgDeclareInst>(&BI))
1544          Finder.processDeclare(*M, DDI);
1545      }
1546    }
1547  }
1548
1549  // After we've found all our debuginfo, rewrite all subprograms to point to
1550  // the same `DICompileUnit`.
1551  for (auto &F : Finder.subprograms()) {
1552    F->replaceUnit(Unit);
1553  }
1554
1555  // Erase any other references to other `DICompileUnit` instances, the verifier
1556  // will later ensure that we don't actually have any other stale references to
1557  // worry about.
1558  auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1559  MD->clearOperands();
1560  MD->addOperand(Unit);
1561}