fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #69444 - pietroalbini:stable-next-final-v2-new, r=pietroalbinif3e1a954d2e

1.41.1
39.4 KiB1197 linesraw
1#include <stdio.h>
2
3#include <vector>
4#include <set>
5
6#include "rustllvm.h"
7
8#include "llvm/Analysis/TargetLibraryInfo.h"
9#include "llvm/Analysis/TargetTransformInfo.h"
10#include "llvm/CodeGen/TargetSubtargetInfo.h"
11#include "llvm/IR/AutoUpgrade.h"
12#include "llvm/IR/AssemblyAnnotationWriter.h"
13#include "llvm/IR/IntrinsicInst.h"
14#include "llvm/Support/CBindingWrapping.h"
15#include "llvm/Support/FileSystem.h"
16#include "llvm/Support/Host.h"
17#include "llvm/Target/TargetMachine.h"
18#include "llvm/Transforms/IPO/PassManagerBuilder.h"
19#include "llvm/Transforms/IPO/AlwaysInliner.h"
20#include "llvm/Transforms/IPO/FunctionImport.h"
21#include "llvm/Transforms/Utils/FunctionImportUtils.h"
22#include "llvm/LTO/LTO.h"
23#include "llvm-c/Transforms/PassManagerBuilder.h"
24
25#include "llvm/Transforms/Instrumentation.h"
26#if LLVM_VERSION_GE(9, 0)
27#include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
28#endif
29#if LLVM_VERSION_GE(8, 0)
30#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
31#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
32#endif
33
34using namespace llvm;
35using namespace llvm::legacy;
36
37typedef struct LLVMOpaquePass *LLVMPassRef;
38typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
39
40DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
41DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
42DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder,
43                                   LLVMPassManagerBuilderRef)
44
45extern "C" void LLVMInitializePasses() {
46  PassRegistry &Registry = *PassRegistry::getPassRegistry();
47  initializeCore(Registry);
48  initializeCodeGen(Registry);
49  initializeScalarOpts(Registry);
50  initializeVectorization(Registry);
51  initializeIPO(Registry);
52  initializeAnalysis(Registry);
53  initializeTransformUtils(Registry);
54  initializeInstCombine(Registry);
55  initializeInstrumentation(Registry);
56  initializeTarget(Registry);
57}
58
59enum class LLVMRustPassKind {
60  Other,
61  Function,
62  Module,
63};
64
65static LLVMRustPassKind toRust(PassKind Kind) {
66  switch (Kind) {
67  case PT_Function:
68    return LLVMRustPassKind::Function;
69  case PT_Module:
70    return LLVMRustPassKind::Module;
71  default:
72    return LLVMRustPassKind::Other;
73  }
74}
75
76extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
77  StringRef SR(PassName);
78  PassRegistry *PR = PassRegistry::getPassRegistry();
79
80  const PassInfo *PI = PR->getPassInfo(SR);
81  if (PI) {
82    return wrap(PI->createPass());
83  }
84  return nullptr;
85}
86
87extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
88  const bool CompileKernel = false;
89
90  return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover));
91}
92
93extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
94  const bool CompileKernel = false;
95
96#if LLVM_VERSION_GE(9, 0)
97  return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover));
98#else
99  return wrap(createAddressSanitizerModulePass(CompileKernel, Recover));
100#endif
101}
102
103extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) {
104#if LLVM_VERSION_GE(9, 0)
105  const bool CompileKernel = false;
106
107  return wrap(createMemorySanitizerLegacyPassPass(
108      MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel}));
109#elif LLVM_VERSION_GE(8, 0)
110  return wrap(createMemorySanitizerLegacyPassPass(TrackOrigins, Recover));
111#else
112  return wrap(createMemorySanitizerPass(TrackOrigins, Recover));
113#endif
114}
115
116extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() {
117#if LLVM_VERSION_GE(8, 0)
118  return wrap(createThreadSanitizerLegacyPassPass());
119#else
120  return wrap(createThreadSanitizerPass());
121#endif
122}
123
124extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) {
125  assert(RustPass);
126  Pass *Pass = unwrap(RustPass);
127  return toRust(Pass->getPassKind());
128}
129
130extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
131  assert(RustPass);
132  Pass *Pass = unwrap(RustPass);
133  PassManagerBase *PMB = unwrap(PMR);
134  PMB->add(Pass);
135}
136
137extern "C"
138void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
139  LLVMPassManagerBuilderRef PMBR,
140  LLVMPassManagerRef PMR
141) {
142  unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
143}
144
145extern "C"
146void LLVMRustAddLastExtensionPasses(
147    LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
148  auto AddExtensionPasses = [Passes, NumPasses](
149      const PassManagerBuilder &Builder, PassManagerBase &PM) {
150    for (size_t I = 0; I < NumPasses; I++) {
151      PM.add(unwrap(Passes[I]));
152    }
153  };
154  // Add the passes to both of the pre-finalization extension points,
155  // so they are run for optimized and non-optimized builds.
156  unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
157                             AddExtensionPasses);
158  unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
159                             AddExtensionPasses);
160}
161
162#ifdef LLVM_COMPONENT_X86
163#define SUBTARGET_X86 SUBTARGET(X86)
164#else
165#define SUBTARGET_X86
166#endif
167
168#ifdef LLVM_COMPONENT_ARM
169#define SUBTARGET_ARM SUBTARGET(ARM)
170#else
171#define SUBTARGET_ARM
172#endif
173
174#ifdef LLVM_COMPONENT_AARCH64
175#define SUBTARGET_AARCH64 SUBTARGET(AArch64)
176#else
177#define SUBTARGET_AARCH64
178#endif
179
180#ifdef LLVM_COMPONENT_MIPS
181#define SUBTARGET_MIPS SUBTARGET(Mips)
182#else
183#define SUBTARGET_MIPS
184#endif
185
186#ifdef LLVM_COMPONENT_POWERPC
187#define SUBTARGET_PPC SUBTARGET(PPC)
188#else
189#define SUBTARGET_PPC
190#endif
191
192#ifdef LLVM_COMPONENT_SYSTEMZ
193#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
194#else
195#define SUBTARGET_SYSTEMZ
196#endif
197
198#ifdef LLVM_COMPONENT_MSP430
199#define SUBTARGET_MSP430 SUBTARGET(MSP430)
200#else
201#define SUBTARGET_MSP430
202#endif
203
204#ifdef LLVM_COMPONENT_RISCV
205#define SUBTARGET_RISCV SUBTARGET(RISCV)
206#else
207#define SUBTARGET_RISCV
208#endif
209
210#ifdef LLVM_COMPONENT_SPARC
211#define SUBTARGET_SPARC SUBTARGET(Sparc)
212#else
213#define SUBTARGET_SPARC
214#endif
215
216#ifdef LLVM_COMPONENT_HEXAGON
217#define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
218#else
219#define SUBTARGET_HEXAGON
220#endif
221
222#define GEN_SUBTARGETS                                                         \
223  SUBTARGET_X86                                                                \
224  SUBTARGET_ARM                                                                \
225  SUBTARGET_AARCH64                                                            \
226  SUBTARGET_MIPS                                                               \
227  SUBTARGET_PPC                                                                \
228  SUBTARGET_SYSTEMZ                                                            \
229  SUBTARGET_MSP430                                                             \
230  SUBTARGET_SPARC                                                              \
231  SUBTARGET_HEXAGON                                                            \
232  SUBTARGET_RISCV                                                              \
233
234#define SUBTARGET(x)                                                           \
235  namespace llvm {                                                             \
236  extern const SubtargetFeatureKV x##FeatureKV[];                              \
237  extern const SubtargetFeatureKV x##SubTypeKV[];                              \
238  }
239
240GEN_SUBTARGETS
241#undef SUBTARGET
242
243extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
244                                   const char *Feature) {
245  TargetMachine *Target = unwrap(TM);
246  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
247  return MCInfo->checkFeatures(std::string("+") + Feature);
248}
249
250enum class LLVMRustCodeModel {
251  Other,
252  Small,
253  Kernel,
254  Medium,
255  Large,
256  None,
257};
258
259static CodeModel::Model fromRust(LLVMRustCodeModel Model) {
260  switch (Model) {
261  case LLVMRustCodeModel::Small:
262    return CodeModel::Small;
263  case LLVMRustCodeModel::Kernel:
264    return CodeModel::Kernel;
265  case LLVMRustCodeModel::Medium:
266    return CodeModel::Medium;
267  case LLVMRustCodeModel::Large:
268    return CodeModel::Large;
269  default:
270    report_fatal_error("Bad CodeModel.");
271  }
272}
273
274enum class LLVMRustCodeGenOptLevel {
275  Other,
276  None,
277  Less,
278  Default,
279  Aggressive,
280};
281
282static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
283  switch (Level) {
284  case LLVMRustCodeGenOptLevel::None:
285    return CodeGenOpt::None;
286  case LLVMRustCodeGenOptLevel::Less:
287    return CodeGenOpt::Less;
288  case LLVMRustCodeGenOptLevel::Default:
289    return CodeGenOpt::Default;
290  case LLVMRustCodeGenOptLevel::Aggressive:
291    return CodeGenOpt::Aggressive;
292  default:
293    report_fatal_error("Bad CodeGenOptLevel.");
294  }
295}
296
297enum class LLVMRustRelocMode {
298  Default,
299  Static,
300  PIC,
301  DynamicNoPic,
302  ROPI,
303  RWPI,
304  ROPIRWPI,
305};
306
307static Optional<Reloc::Model> fromRust(LLVMRustRelocMode RustReloc) {
308  switch (RustReloc) {
309  case LLVMRustRelocMode::Default:
310    return None;
311  case LLVMRustRelocMode::Static:
312    return Reloc::Static;
313  case LLVMRustRelocMode::PIC:
314    return Reloc::PIC_;
315  case LLVMRustRelocMode::DynamicNoPic:
316    return Reloc::DynamicNoPIC;
317  case LLVMRustRelocMode::ROPI:
318    return Reloc::ROPI;
319  case LLVMRustRelocMode::RWPI:
320    return Reloc::RWPI;
321  case LLVMRustRelocMode::ROPIRWPI:
322    return Reloc::ROPI_RWPI;
323  }
324  report_fatal_error("Bad RelocModel.");
325}
326
327#ifdef LLVM_RUSTLLVM
328/// getLongestEntryLength - Return the length of the longest entry in the table.
329template<typename KV>
330static size_t getLongestEntryLength(ArrayRef<KV> Table) {
331  size_t MaxLen = 0;
332  for (auto &I : Table)
333    MaxLen = std::max(MaxLen, std::strlen(I.Key));
334  return MaxLen;
335}
336
337extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
338  const TargetMachine *Target = unwrap(TM);
339  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
340  const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
341  const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
342  const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
343  unsigned MaxCPULen = getLongestEntryLength(CPUTable);
344
345  printf("Available CPUs for this target:\n");
346  if (HostArch == TargetArch) {
347    const StringRef HostCPU = sys::getHostCPUName();
348    printf("    %-*s - Select the CPU of the current host (currently %.*s).\n",
349      MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
350  }
351  for (auto &CPU : CPUTable)
352    printf("    %-*s\n", MaxCPULen, CPU.Key);
353  printf("\n");
354}
355
356extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) {
357  const TargetMachine *Target = unwrap(TM);
358  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
359  const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
360  unsigned MaxFeatLen = getLongestEntryLength(FeatTable);
361
362  printf("Available features for this target:\n");
363  for (auto &Feature : FeatTable)
364    printf("    %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc);
365  printf("\n");
366
367  printf("Use +feature to enable a feature, or -feature to disable it.\n"
368         "For example, rustc -C -target-cpu=mycpu -C "
369         "target-feature=+feature1,-feature2\n\n");
370}
371
372#else
373
374extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
375  printf("Target CPU help is not supported by this LLVM version.\n\n");
376}
377
378extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) {
379  printf("Target features help is not supported by this LLVM version.\n\n");
380}
381#endif
382
383extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
384  StringRef Name = sys::getHostCPUName();
385  *len = Name.size();
386  return Name.data();
387}
388
389extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
390    const char *TripleStr, const char *CPU, const char *Feature,
391    const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocMode RustReloc,
392    LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
393    bool PositionIndependentExecutable, bool FunctionSections,
394    bool DataSections,
395    bool TrapUnreachable,
396    bool Singlethread,
397    bool AsmComments,
398    bool EmitStackSizeSection,
399    bool RelaxELFRelocations) {
400
401  auto OptLevel = fromRust(RustOptLevel);
402  auto RM = fromRust(RustReloc);
403
404  std::string Error;
405  Triple Trip(Triple::normalize(TripleStr));
406  const llvm::Target *TheTarget =
407      TargetRegistry::lookupTarget(Trip.getTriple(), Error);
408  if (TheTarget == nullptr) {
409    LLVMRustSetLastError(Error.c_str());
410    return nullptr;
411  }
412
413  TargetOptions Options;
414
415  Options.FloatABIType = FloatABI::Default;
416  if (UseSoftFloat) {
417    Options.FloatABIType = FloatABI::Soft;
418  }
419  Options.DataSections = DataSections;
420  Options.FunctionSections = FunctionSections;
421  Options.MCOptions.AsmVerbose = AsmComments;
422  Options.MCOptions.PreserveAsmComments = AsmComments;
423  Options.MCOptions.ABIName = ABIStr;
424  Options.RelaxELFRelocations = RelaxELFRelocations;
425
426  if (TrapUnreachable) {
427    // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
428    // This limits the extent of possible undefined behavior in some cases, as
429    // it prevents control flow from "falling through" into whatever code
430    // happens to be laid out next in memory.
431    Options.TrapUnreachable = true;
432  }
433
434  if (Singlethread) {
435    Options.ThreadModel = ThreadModel::Single;
436  }
437
438  Options.EmitStackSizeSection = EmitStackSizeSection;
439
440  Optional<CodeModel::Model> CM;
441  if (RustCM != LLVMRustCodeModel::None)
442    CM = fromRust(RustCM);
443  TargetMachine *TM = TheTarget->createTargetMachine(
444      Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
445  return wrap(TM);
446}
447
448extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
449  delete unwrap(TM);
450}
451
452extern "C" void LLVMRustConfigurePassManagerBuilder(
453    LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
454    bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
455    const char* PGOGenPath, const char* PGOUsePath) {
456  unwrap(PMBR)->MergeFunctions = MergeFunctions;
457  unwrap(PMBR)->SLPVectorize = SLPVectorize;
458  unwrap(PMBR)->OptLevel = fromRust(OptLevel);
459  unwrap(PMBR)->LoopVectorize = LoopVectorize;
460  unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
461
462  if (PGOGenPath) {
463    assert(!PGOUsePath);
464    unwrap(PMBR)->EnablePGOInstrGen = true;
465    unwrap(PMBR)->PGOInstrGen = PGOGenPath;
466  }
467  if (PGOUsePath) {
468    assert(!PGOGenPath);
469    unwrap(PMBR)->PGOInstrUse = PGOUsePath;
470  }
471}
472
473// Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
474// field of a PassManagerBuilder, we expose our own method of doing so.
475extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
476                                              LLVMModuleRef M,
477                                              bool DisableSimplifyLibCalls) {
478  Triple TargetTriple(unwrap(M)->getTargetTriple());
479  TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
480  if (DisableSimplifyLibCalls)
481    TLI->disableAllFunctions();
482  unwrap(PMBR)->LibraryInfo = TLI;
483}
484
485// Unfortunately, the LLVM C API doesn't provide a way to create the
486// TargetLibraryInfo pass, so we use this method to do so.
487extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
488                                       bool DisableSimplifyLibCalls) {
489  Triple TargetTriple(unwrap(M)->getTargetTriple());
490  TargetLibraryInfoImpl TLII(TargetTriple);
491  if (DisableSimplifyLibCalls)
492    TLII.disableAllFunctions();
493  unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
494}
495
496// Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
497// all the functions in a module, so we do that manually here. You'll find
498// similar code in clang's BackendUtil.cpp file.
499extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
500                                               LLVMModuleRef M) {
501  llvm::legacy::FunctionPassManager *P =
502      unwrap<llvm::legacy::FunctionPassManager>(PMR);
503  P->doInitialization();
504
505  // Upgrade all calls to old intrinsics first.
506  for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
507    UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
508
509  for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
510       ++I)
511    if (!I->isDeclaration())
512      P->run(*I);
513
514  P->doFinalization();
515}
516
517extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
518  // Initializing the command-line options more than once is not allowed. So,
519  // check if they've already been initialized.  (This could happen if we're
520  // being called from rustpkg, for example). If the arguments change, then
521  // that's just kinda unfortunate.
522  static bool Initialized = false;
523  if (Initialized)
524    return;
525  Initialized = true;
526  cl::ParseCommandLineOptions(Argc, Argv);
527}
528
529enum class LLVMRustFileType {
530  Other,
531  AssemblyFile,
532  ObjectFile,
533};
534
535static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) {
536  switch (Type) {
537  case LLVMRustFileType::AssemblyFile:
538    return TargetMachine::CGFT_AssemblyFile;
539  case LLVMRustFileType::ObjectFile:
540    return TargetMachine::CGFT_ObjectFile;
541  default:
542    report_fatal_error("Bad FileType.");
543  }
544}
545
546extern "C" LLVMRustResult
547LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
548                        LLVMModuleRef M, const char *Path,
549                        LLVMRustFileType RustFileType) {
550  llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
551  auto FileType = fromRust(RustFileType);
552
553  std::string ErrorInfo;
554  std::error_code EC;
555  raw_fd_ostream OS(Path, EC, sys::fs::F_None);
556  if (EC)
557    ErrorInfo = EC.message();
558  if (ErrorInfo != "") {
559    LLVMRustSetLastError(ErrorInfo.c_str());
560    return LLVMRustResult::Failure;
561  }
562
563  buffer_ostream BOS(OS);
564  unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
565  PM->run(*unwrap(M));
566
567  // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
568  // stream (OS), so the only real safe place to delete this is here? Don't we
569  // wish this was written in Rust?
570  LLVMDisposePassManager(PMR);
571  return LLVMRustResult::Success;
572}
573
574
575// Callback to demangle function name
576// Parameters:
577// * name to be demangled
578// * name len
579// * output buffer
580// * output buffer len
581// Returns len of demangled string, or 0 if demangle failed.
582typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
583
584
585namespace {
586
587class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
588  DemangleFn Demangle;
589  std::vector<char> Buf;
590
591public:
592  RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
593
594  // Return empty string if demangle failed
595  // or if name does not need to be demangled
596  StringRef CallDemangle(StringRef name) {
597    if (!Demangle) {
598      return StringRef();
599    }
600
601    if (Buf.size() < name.size() * 2) {
602      // Semangled name usually shorter than mangled,
603      // but allocate twice as much memory just in case
604      Buf.resize(name.size() * 2);
605    }
606
607    auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
608    if (!R) {
609      // Demangle failed.
610      return StringRef();
611    }
612
613    auto Demangled = StringRef(Buf.data(), R);
614    if (Demangled == name) {
615      // Do not print anything if demangled name is equal to mangled.
616      return StringRef();
617    }
618
619    return Demangled;
620  }
621
622  void emitFunctionAnnot(const Function *F,
623                         formatted_raw_ostream &OS) override {
624    StringRef Demangled = CallDemangle(F->getName());
625    if (Demangled.empty()) {
626        return;
627    }
628
629    OS << "; " << Demangled << "\n";
630  }
631
632  void emitInstructionAnnot(const Instruction *I,
633                            formatted_raw_ostream &OS) override {
634    const char *Name;
635    const Value *Value;
636    if (const CallInst *CI = dyn_cast<CallInst>(I)) {
637      Name = "call";
638      Value = CI->getCalledValue();
639    } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
640      Name = "invoke";
641      Value = II->getCalledValue();
642    } else {
643      // Could demangle more operations, e. g.
644      // `store %place, @function`.
645      return;
646    }
647
648    if (!Value->hasName()) {
649      return;
650    }
651
652    StringRef Demangled = CallDemangle(Value->getName());
653    if (Demangled.empty()) {
654      return;
655    }
656
657    OS << "; " << Name << " " << Demangled << "\n";
658  }
659};
660
661} // namespace
662
663extern "C" LLVMRustResult
664LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
665  std::string ErrorInfo;
666  std::error_code EC;
667  raw_fd_ostream OS(Path, EC, sys::fs::F_None);
668  if (EC)
669    ErrorInfo = EC.message();
670  if (ErrorInfo != "") {
671    LLVMRustSetLastError(ErrorInfo.c_str());
672    return LLVMRustResult::Failure;
673  }
674
675  RustAssemblyAnnotationWriter AAW(Demangle);
676  formatted_raw_ostream FOS(OS);
677  unwrap(M)->print(FOS, &AAW);
678
679  return LLVMRustResult::Success;
680}
681
682extern "C" void LLVMRustPrintPasses() {
683  LLVMInitializePasses();
684  struct MyListener : PassRegistrationListener {
685    void passEnumerate(const PassInfo *Info) {
686      StringRef PassArg = Info->getPassArgument();
687      StringRef PassName = Info->getPassName();
688      if (!PassArg.empty()) {
689        // These unsigned->signed casts could theoretically overflow, but
690        // realistically never will (and even if, the result is implementation
691        // defined rather plain UB).
692        printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
693               (int)PassName.size(), PassName.data());
694      }
695    }
696  } Listener;
697
698  PassRegistry *PR = PassRegistry::getPassRegistry();
699  PR->enumerateWith(&Listener);
700}
701
702extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
703                                            bool AddLifetimes) {
704  unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
705}
706
707extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
708                                           size_t Len) {
709  llvm::legacy::PassManager passes;
710
711  auto PreserveFunctions = [=](const GlobalValue &GV) {
712    for (size_t I = 0; I < Len; I++) {
713      if (GV.getName() == Symbols[I]) {
714        return true;
715      }
716    }
717    return false;
718  };
719
720  passes.add(llvm::createInternalizePass(PreserveFunctions));
721
722  passes.run(*unwrap(M));
723}
724
725extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) {
726  for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E;
727       ++GV) {
728    GV->setDoesNotThrow();
729    Function *F = dyn_cast<Function>(GV);
730    if (F == nullptr)
731      continue;
732
733    for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
734      for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) {
735        if (isa<InvokeInst>(I)) {
736          InvokeInst *CI = cast<InvokeInst>(I);
737          CI->setDoesNotThrow();
738        }
739      }
740    }
741  }
742}
743
744extern "C" void
745LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
746                                       LLVMTargetMachineRef TMR) {
747  TargetMachine *Target = unwrap(TMR);
748  unwrap(Module)->setDataLayout(Target->createDataLayout());
749}
750
751extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
752  unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
753}
754
755extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
756  unwrap(M)->setPIELevel(PIELevel::Level::Large);
757}
758
759// Here you'll find an implementation of ThinLTO as used by the Rust compiler
760// right now. This ThinLTO support is only enabled on "recent ish" versions of
761// LLVM, and otherwise it's just blanket rejected from other compilers.
762//
763// Most of this implementation is straight copied from LLVM. At the time of
764// this writing it wasn't *quite* suitable to reuse more code from upstream
765// for our purposes, but we should strive to upstream this support once it's
766// ready to go! I figure we may want a bit of testing locally first before
767// sending this upstream to LLVM. I hear though they're quite eager to receive
768// feedback like this!
769//
770// If you're reading this code and wondering "what in the world" or you're
771// working "good lord by LLVM upgrade is *still* failing due to these bindings"
772// then fear not! (ok maybe fear a little). All code here is mostly based
773// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
774//
775// You'll find that the general layout here roughly corresponds to the `run`
776// method in that file as well as `ProcessThinLTOModule`. Functions are
777// specifically commented below as well, but if you're updating this code
778// or otherwise trying to understand it, the LLVM source will be useful in
779// interpreting the mysteries within.
780//
781// Otherwise I'll apologize in advance, it probably requires a relatively
782// significant investment on your part to "truly understand" what's going on
783// here. Not saying I do myself, but it took me awhile staring at LLVM's source
784// and various online resources about ThinLTO to make heads or tails of all
785// this.
786
787// This is a shared data structure which *must* be threadsafe to share
788// read-only amongst threads. This also corresponds basically to the arguments
789// of the `ProcessThinLTOModule` function in the LLVM source.
790struct LLVMRustThinLTOData {
791  // The combined index that is the global analysis over all modules we're
792  // performing ThinLTO for. This is mostly managed by LLVM.
793  ModuleSummaryIndex Index;
794
795  // All modules we may look at, stored as in-memory serialized versions. This
796  // is later used when inlining to ensure we can extract any module to inline
797  // from.
798  StringMap<MemoryBufferRef> ModuleMap;
799
800  // A set that we manage of everything we *don't* want internalized. Note that
801  // this includes all transitive references right now as well, but it may not
802  // always!
803  DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
804
805  // Not 100% sure what these are, but they impact what's internalized and
806  // what's inlined across modules, I believe.
807  StringMap<FunctionImporter::ImportMapTy> ImportLists;
808  StringMap<FunctionImporter::ExportSetTy> ExportLists;
809  StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
810
811  LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
812};
813
814// Just an argument to the `LLVMRustCreateThinLTOData` function below.
815struct LLVMRustThinLTOModule {
816  const char *identifier;
817  const char *data;
818  size_t len;
819};
820
821// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
822// does.
823static const GlobalValueSummary *
824getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
825  auto StrongDefForLinker = llvm::find_if(
826      GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
827        auto Linkage = Summary->linkage();
828        return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
829               !GlobalValue::isWeakForLinker(Linkage);
830      });
831  if (StrongDefForLinker != GVSummaryList.end())
832    return StrongDefForLinker->get();
833
834  auto FirstDefForLinker = llvm::find_if(
835      GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
836        auto Linkage = Summary->linkage();
837        return !GlobalValue::isAvailableExternallyLinkage(Linkage);
838      });
839  if (FirstDefForLinker == GVSummaryList.end())
840    return nullptr;
841  return FirstDefForLinker->get();
842}
843
844// The main entry point for creating the global ThinLTO analysis. The structure
845// here is basically the same as before threads are spawned in the `run`
846// function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
847extern "C" LLVMRustThinLTOData*
848LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
849                          int num_modules,
850                          const char **preserved_symbols,
851                          int num_symbols) {
852  auto Ret = llvm::make_unique<LLVMRustThinLTOData>();
853
854  // Load each module's summary and merge it into one combined index
855  for (int i = 0; i < num_modules; i++) {
856    auto module = &modules[i];
857    StringRef buffer(module->data, module->len);
858    MemoryBufferRef mem_buffer(buffer, module->identifier);
859
860    Ret->ModuleMap[module->identifier] = mem_buffer;
861
862    if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
863      LLVMRustSetLastError(toString(std::move(Err)).c_str());
864      return nullptr;
865    }
866  }
867
868  // Collect for each module the list of function it defines (GUID -> Summary)
869  Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
870
871  // Convert the preserved symbols set from string to GUID, this is then needed
872  // for internalization.
873  for (int i = 0; i < num_symbols; i++) {
874    auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
875    Ret->GUIDPreservedSymbols.insert(GUID);
876  }
877
878  // Collect the import/export lists for all modules from the call-graph in the
879  // combined index
880  //
881  // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
882  auto deadIsPrevailing = [&](GlobalValue::GUID G) {
883    return PrevailingType::Unknown;
884  };
885#if LLVM_VERSION_GE(8, 0)
886  // We don't have a complete picture in our use of ThinLTO, just our immediate
887  // crate, so we need `ImportEnabled = false` to limit internalization.
888  // Otherwise, we sometimes lose `static` values -- see #60184.
889  computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
890                                  deadIsPrevailing, /* ImportEnabled = */ false);
891#else
892  computeDeadSymbols(Ret->Index, Ret->GUIDPreservedSymbols, deadIsPrevailing);
893#endif
894  ComputeCrossModuleImport(
895    Ret->Index,
896    Ret->ModuleToDefinedGVSummaries,
897    Ret->ImportLists,
898    Ret->ExportLists
899  );
900
901  // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
902  // impacts the caching.
903  //
904  // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
905  // being lifted from `lib/LTO/LTO.cpp` as well
906  StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
907  DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
908  for (auto &I : Ret->Index) {
909    if (I.second.SummaryList.size() > 1)
910      PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
911  }
912  auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
913    const auto &Prevailing = PrevailingCopy.find(GUID);
914    if (Prevailing == PrevailingCopy.end())
915      return true;
916    return Prevailing->second == S;
917  };
918  auto recordNewLinkage = [&](StringRef ModuleIdentifier,
919                              GlobalValue::GUID GUID,
920                              GlobalValue::LinkageTypes NewLinkage) {
921    ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
922  };
923#if LLVM_VERSION_GE(9, 0)
924  thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage,
925                                  Ret->GUIDPreservedSymbols);
926#elif LLVM_VERSION_GE(8, 0)
927  thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage);
928#else
929  thinLTOResolveWeakForLinkerInIndex(Ret->Index, isPrevailing, recordNewLinkage);
930#endif
931
932  // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
933  // callback below. This callback below will dictate the linkage for all
934  // summaries in the index, and we basically just only want to ensure that dead
935  // symbols are internalized. Otherwise everything that's already external
936  // linkage will stay as external, and internal will stay as internal.
937  std::set<GlobalValue::GUID> ExportedGUIDs;
938  for (auto &List : Ret->Index) {
939    for (auto &GVS: List.second.SummaryList) {
940      if (GlobalValue::isLocalLinkage(GVS->linkage()))
941        continue;
942      auto GUID = GVS->getOriginalName();
943      if (GVS->flags().Live)
944        ExportedGUIDs.insert(GUID);
945    }
946  }
947  auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
948    const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
949    return (ExportList != Ret->ExportLists.end() &&
950      ExportList->second.count(GUID)) ||
951      ExportedGUIDs.count(GUID);
952  };
953  thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported);
954
955  return Ret.release();
956}
957
958extern "C" void
959LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
960  delete Data;
961}
962
963// Below are the various passes that happen *per module* when doing ThinLTO.
964//
965// In other words, these are the functions that are all run concurrently
966// with one another, one per module. The passes here correspond to the analysis
967// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
968// `ProcessThinLTOModule` function. Here they're split up into separate steps
969// so rustc can save off the intermediate bytecode between each step.
970
971extern "C" bool
972LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
973  Module &Mod = *unwrap(M);
974  if (renameModuleForThinLTO(Mod, Data->Index)) {
975    LLVMRustSetLastError("renameModuleForThinLTO failed");
976    return false;
977  }
978  return true;
979}
980
981extern "C" bool
982LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
983  Module &Mod = *unwrap(M);
984  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
985#if LLVM_VERSION_GE(8, 0)
986  thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
987#else
988  thinLTOResolveWeakForLinkerModule(Mod, DefinedGlobals);
989#endif
990  return true;
991}
992
993extern "C" bool
994LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
995  Module &Mod = *unwrap(M);
996  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
997  thinLTOInternalizeModule(Mod, DefinedGlobals);
998  return true;
999}
1000
1001extern "C" bool
1002LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1003  Module &Mod = *unwrap(M);
1004
1005  const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1006  auto Loader = [&](StringRef Identifier) {
1007    const auto &Memory = Data->ModuleMap.lookup(Identifier);
1008    auto &Context = Mod.getContext();
1009    auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1010
1011    if (!MOrErr)
1012      return MOrErr;
1013
1014    // The rest of this closure is a workaround for
1015    // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1016    // we accidentally import wasm custom sections into different modules,
1017    // duplicating them by in the final output artifact.
1018    //
1019    // The issue is worked around here by manually removing the
1020    // `wasm.custom_sections` named metadata node from any imported module. This
1021    // we know isn't used by any optimization pass so there's no need for it to
1022    // be imported.
1023    //
1024    // Note that the metadata is currently lazily loaded, so we materialize it
1025    // here before looking up if there's metadata inside. The `FunctionImporter`
1026    // will immediately materialize metadata anyway after an import, so this
1027    // shouldn't be a perf hit.
1028    if (Error Err = (*MOrErr)->materializeMetadata()) {
1029      Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1030      return Ret;
1031    }
1032
1033    auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1034    if (WasmCustomSections)
1035      WasmCustomSections->eraseFromParent();
1036
1037    return MOrErr;
1038  };
1039  FunctionImporter Importer(Data->Index, Loader);
1040  Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1041  if (!Result) {
1042    LLVMRustSetLastError(toString(Result.takeError()).c_str());
1043    return false;
1044  }
1045  return true;
1046}
1047
1048extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload
1049                                                      const char*, // importing module name
1050                                                      const char*); // imported module name
1051
1052// Calls `module_name_callback` for each module import done by ThinLTO.
1053// The callback is provided with regular null-terminated C strings.
1054extern "C" void
1055LLVMRustGetThinLTOModuleImports(const LLVMRustThinLTOData *data,
1056                                LLVMRustModuleNameCallback module_name_callback,
1057                                void* callback_payload) {
1058  for (const auto& importing_module : data->ImportLists) {
1059    const std::string importing_module_id = importing_module.getKey().str();
1060    const auto& imports = importing_module.getValue();
1061    for (const auto& imported_module : imports) {
1062      const std::string imported_module_id = imported_module.getKey().str();
1063      module_name_callback(callback_payload,
1064                           importing_module_id.c_str(),
1065                           imported_module_id.c_str());
1066    }
1067  }
1068}
1069
1070// This struct and various functions are sort of a hack right now, but the
1071// problem is that we've got in-memory LLVM modules after we generate and
1072// optimize all codegen-units for one compilation in rustc. To be compatible
1073// with the LTO support above we need to serialize the modules plus their
1074// ThinLTO summary into memory.
1075//
1076// This structure is basically an owned version of a serialize module, with
1077// a ThinLTO summary attached.
1078struct LLVMRustThinLTOBuffer {
1079  std::string data;
1080};
1081
1082extern "C" LLVMRustThinLTOBuffer*
1083LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
1084  auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>();
1085  {
1086    raw_string_ostream OS(Ret->data);
1087    {
1088      legacy::PassManager PM;
1089      PM.add(createWriteThinLTOBitcodePass(OS));
1090      PM.run(*unwrap(M));
1091    }
1092  }
1093  return Ret.release();
1094}
1095
1096extern "C" void
1097LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1098  delete Buffer;
1099}
1100
1101extern "C" const void*
1102LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1103  return Buffer->data.data();
1104}
1105
1106extern "C" size_t
1107LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1108  return Buffer->data.length();
1109}
1110
1111// This is what we used to parse upstream bitcode for actual ThinLTO
1112// processing.  We'll call this once per module optimized through ThinLTO, and
1113// it'll be called concurrently on many threads.
1114extern "C" LLVMModuleRef
1115LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1116                           const char *data,
1117                           size_t len,
1118                           const char *identifier) {
1119  StringRef Data(data, len);
1120  MemoryBufferRef Buffer(Data, identifier);
1121  unwrap(Context)->enableDebugTypeODRUniquing();
1122  Expected<std::unique_ptr<Module>> SrcOrError =
1123      parseBitcodeFile(Buffer, *unwrap(Context));
1124  if (!SrcOrError) {
1125    LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1126    return nullptr;
1127  }
1128  return wrap(std::move(*SrcOrError).release());
1129}
1130
1131// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1132// the comment in `back/lto.rs` for why this exists.
1133extern "C" void
1134LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1135                                DICompileUnit **A,
1136                                DICompileUnit **B) {
1137  Module *M = unwrap(Mod);
1138  DICompileUnit **Cur = A;
1139  DICompileUnit **Next = B;
1140  for (DICompileUnit *CU : M->debug_compile_units()) {
1141    *Cur = CU;
1142    Cur = Next;
1143    Next = nullptr;
1144    if (Cur == nullptr)
1145      break;
1146  }
1147}
1148
1149// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1150// the comment in `back/lto.rs` for why this exists.
1151extern "C" void
1152LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1153  Module *M = unwrap(Mod);
1154
1155  // If the original source module didn't have a `DICompileUnit` then try to
1156  // merge all the existing compile units. If there aren't actually any though
1157  // then there's not much for us to do so return.
1158  if (Unit == nullptr) {
1159    for (DICompileUnit *CU : M->debug_compile_units()) {
1160      Unit = CU;
1161      break;
1162    }
1163    if (Unit == nullptr)
1164      return;
1165  }
1166
1167  // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1168  // process it recursively. Note that we specifically iterate over instructions
1169  // to ensure we feed everything into it.
1170  DebugInfoFinder Finder;
1171  Finder.processModule(*M);
1172  for (Function &F : M->functions()) {
1173    for (auto &FI : F) {
1174      for (Instruction &BI : FI) {
1175        if (auto Loc = BI.getDebugLoc())
1176          Finder.processLocation(*M, Loc);
1177        if (auto DVI = dyn_cast<DbgValueInst>(&BI))
1178          Finder.processValue(*M, DVI);
1179        if (auto DDI = dyn_cast<DbgDeclareInst>(&BI))
1180          Finder.processDeclare(*M, DDI);
1181      }
1182    }
1183  }
1184
1185  // After we've found all our debuginfo, rewrite all subprograms to point to
1186  // the same `DICompileUnit`.
1187  for (auto &F : Finder.subprograms()) {
1188    F->replaceUnit(Unit);
1189  }
1190
1191  // Erase any other references to other `DICompileUnit` instances, the verifier
1192  // will later ensure that we don't actually have any other stale references to
1193  // worry about.
1194  auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1195  MD->clearOperands();
1196  MD->addOperand(Unit);
1197}