fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #67350 - Mark-Simulacrum:stable-next, r=Mark-Simulacrum73528e339aa

1.40.0
39.4 KiB1205 linesraw
1#include <stdio.h>
2
3#include <vector>
4#include <set>
5
6#include "rustllvm.h"
7
8#include "llvm/Analysis/TargetLibraryInfo.h"
9#include "llvm/Analysis/TargetTransformInfo.h"
10#include "llvm/CodeGen/TargetSubtargetInfo.h"
11#include "llvm/IR/AutoUpgrade.h"
12#include "llvm/IR/AssemblyAnnotationWriter.h"
13#include "llvm/IR/IntrinsicInst.h"
14#include "llvm/Support/CBindingWrapping.h"
15#include "llvm/Support/FileSystem.h"
16#include "llvm/Support/Host.h"
17#include "llvm/Target/TargetMachine.h"
18#include "llvm/Transforms/IPO/PassManagerBuilder.h"
19#include "llvm/Transforms/IPO/AlwaysInliner.h"
20#include "llvm/Transforms/IPO/FunctionImport.h"
21#include "llvm/Transforms/Utils/FunctionImportUtils.h"
22#include "llvm/LTO/LTO.h"
23
24#include "llvm-c/Transforms/PassManagerBuilder.h"
25
26using namespace llvm;
27using namespace llvm::legacy;
28
29typedef struct LLVMOpaquePass *LLVMPassRef;
30typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
31
32DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
33DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
34DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder,
35                                   LLVMPassManagerBuilderRef)
36
37extern "C" void LLVMInitializePasses() {
38  PassRegistry &Registry = *PassRegistry::getPassRegistry();
39  initializeCore(Registry);
40  initializeCodeGen(Registry);
41  initializeScalarOpts(Registry);
42  initializeVectorization(Registry);
43  initializeIPO(Registry);
44  initializeAnalysis(Registry);
45  initializeTransformUtils(Registry);
46  initializeInstCombine(Registry);
47  initializeInstrumentation(Registry);
48  initializeTarget(Registry);
49}
50
51enum class LLVMRustPassKind {
52  Other,
53  Function,
54  Module,
55};
56
57static LLVMRustPassKind toRust(PassKind Kind) {
58  switch (Kind) {
59  case PT_Function:
60    return LLVMRustPassKind::Function;
61  case PT_Module:
62    return LLVMRustPassKind::Module;
63  default:
64    return LLVMRustPassKind::Other;
65  }
66}
67
68extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
69  StringRef SR(PassName);
70  PassRegistry *PR = PassRegistry::getPassRegistry();
71
72  const PassInfo *PI = PR->getPassInfo(SR);
73  if (PI) {
74    return wrap(PI->createPass());
75  }
76  return nullptr;
77}
78
79extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) {
80  assert(RustPass);
81  Pass *Pass = unwrap(RustPass);
82  return toRust(Pass->getPassKind());
83}
84
85extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
86  assert(RustPass);
87  Pass *Pass = unwrap(RustPass);
88  PassManagerBase *PMB = unwrap(PMR);
89  PMB->add(Pass);
90}
91
92extern "C"
93void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
94  LLVMPassManagerBuilderRef PMBR,
95  LLVMPassManagerRef PMR
96) {
97  unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
98}
99
100extern "C"
101void LLVMRustAddLastExtensionPasses(
102    LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
103  auto AddExtensionPasses = [Passes, NumPasses](
104      const PassManagerBuilder &Builder, PassManagerBase &PM) {
105    for (size_t I = 0; I < NumPasses; I++) {
106      PM.add(unwrap(Passes[I]));
107    }
108  };
109  // Add the passes to both of the pre-finalization extension points,
110  // so they are run for optimized and non-optimized builds.
111  unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
112                             AddExtensionPasses);
113  unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
114                             AddExtensionPasses);
115}
116
117#ifdef LLVM_COMPONENT_X86
118#define SUBTARGET_X86 SUBTARGET(X86)
119#else
120#define SUBTARGET_X86
121#endif
122
123#ifdef LLVM_COMPONENT_ARM
124#define SUBTARGET_ARM SUBTARGET(ARM)
125#else
126#define SUBTARGET_ARM
127#endif
128
129#ifdef LLVM_COMPONENT_AARCH64
130#define SUBTARGET_AARCH64 SUBTARGET(AArch64)
131#else
132#define SUBTARGET_AARCH64
133#endif
134
135#ifdef LLVM_COMPONENT_MIPS
136#define SUBTARGET_MIPS SUBTARGET(Mips)
137#else
138#define SUBTARGET_MIPS
139#endif
140
141#ifdef LLVM_COMPONENT_POWERPC
142#define SUBTARGET_PPC SUBTARGET(PPC)
143#else
144#define SUBTARGET_PPC
145#endif
146
147#ifdef LLVM_COMPONENT_SYSTEMZ
148#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
149#else
150#define SUBTARGET_SYSTEMZ
151#endif
152
153#ifdef LLVM_COMPONENT_MSP430
154#define SUBTARGET_MSP430 SUBTARGET(MSP430)
155#else
156#define SUBTARGET_MSP430
157#endif
158
159#ifdef LLVM_COMPONENT_RISCV
160#define SUBTARGET_RISCV SUBTARGET(RISCV)
161#else
162#define SUBTARGET_RISCV
163#endif
164
165#ifdef LLVM_COMPONENT_SPARC
166#define SUBTARGET_SPARC SUBTARGET(Sparc)
167#else
168#define SUBTARGET_SPARC
169#endif
170
171#ifdef LLVM_COMPONENT_HEXAGON
172#define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
173#else
174#define SUBTARGET_HEXAGON
175#endif
176
177#define GEN_SUBTARGETS                                                         \
178  SUBTARGET_X86                                                                \
179  SUBTARGET_ARM                                                                \
180  SUBTARGET_AARCH64                                                            \
181  SUBTARGET_MIPS                                                               \
182  SUBTARGET_PPC                                                                \
183  SUBTARGET_SYSTEMZ                                                            \
184  SUBTARGET_MSP430                                                             \
185  SUBTARGET_SPARC                                                              \
186  SUBTARGET_HEXAGON                                                            \
187  SUBTARGET_RISCV                                                              \
188
189#define SUBTARGET(x)                                                           \
190  namespace llvm {                                                             \
191  extern const SubtargetFeatureKV x##FeatureKV[];                              \
192  extern const SubtargetFeatureKV x##SubTypeKV[];                              \
193  }
194
195GEN_SUBTARGETS
196#undef SUBTARGET
197
198extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
199                                   const char *Feature) {
200  TargetMachine *Target = unwrap(TM);
201  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
202  return MCInfo->checkFeatures(std::string("+") + Feature);
203}
204
205enum class LLVMRustCodeModel {
206  Other,
207  Small,
208  Kernel,
209  Medium,
210  Large,
211  None,
212};
213
214static CodeModel::Model fromRust(LLVMRustCodeModel Model) {
215  switch (Model) {
216  case LLVMRustCodeModel::Small:
217    return CodeModel::Small;
218  case LLVMRustCodeModel::Kernel:
219    return CodeModel::Kernel;
220  case LLVMRustCodeModel::Medium:
221    return CodeModel::Medium;
222  case LLVMRustCodeModel::Large:
223    return CodeModel::Large;
224  default:
225    report_fatal_error("Bad CodeModel.");
226  }
227}
228
229enum class LLVMRustCodeGenOptLevel {
230  Other,
231  None,
232  Less,
233  Default,
234  Aggressive,
235};
236
237static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
238  switch (Level) {
239  case LLVMRustCodeGenOptLevel::None:
240    return CodeGenOpt::None;
241  case LLVMRustCodeGenOptLevel::Less:
242    return CodeGenOpt::Less;
243  case LLVMRustCodeGenOptLevel::Default:
244    return CodeGenOpt::Default;
245  case LLVMRustCodeGenOptLevel::Aggressive:
246    return CodeGenOpt::Aggressive;
247  default:
248    report_fatal_error("Bad CodeGenOptLevel.");
249  }
250}
251
252enum class LLVMRustRelocMode {
253  Default,
254  Static,
255  PIC,
256  DynamicNoPic,
257  ROPI,
258  RWPI,
259  ROPIRWPI,
260};
261
262static Optional<Reloc::Model> fromRust(LLVMRustRelocMode RustReloc) {
263  switch (RustReloc) {
264  case LLVMRustRelocMode::Default:
265    return None;
266  case LLVMRustRelocMode::Static:
267    return Reloc::Static;
268  case LLVMRustRelocMode::PIC:
269    return Reloc::PIC_;
270  case LLVMRustRelocMode::DynamicNoPic:
271    return Reloc::DynamicNoPIC;
272  case LLVMRustRelocMode::ROPI:
273    return Reloc::ROPI;
274  case LLVMRustRelocMode::RWPI:
275    return Reloc::RWPI;
276  case LLVMRustRelocMode::ROPIRWPI:
277    return Reloc::ROPI_RWPI;
278  }
279  report_fatal_error("Bad RelocModel.");
280}
281
282#ifdef LLVM_RUSTLLVM
283/// getLongestEntryLength - Return the length of the longest entry in the table.
284template<typename KV>
285static size_t getLongestEntryLength(ArrayRef<KV> Table) {
286  size_t MaxLen = 0;
287  for (auto &I : Table)
288    MaxLen = std::max(MaxLen, std::strlen(I.Key));
289  return MaxLen;
290}
291
292extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
293  const TargetMachine *Target = unwrap(TM);
294  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
295  const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
296  const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
297  const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
298  unsigned MaxCPULen = getLongestEntryLength(CPUTable);
299
300  printf("Available CPUs for this target:\n");
301  if (HostArch == TargetArch) {
302    const StringRef HostCPU = sys::getHostCPUName();
303    printf("    %-*s - Select the CPU of the current host (currently %.*s).\n",
304      MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
305  }
306  for (auto &CPU : CPUTable)
307    printf("    %-*s\n", MaxCPULen, CPU.Key);
308  printf("\n");
309}
310
311extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) {
312  const TargetMachine *Target = unwrap(TM);
313  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
314  const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
315  unsigned MaxFeatLen = getLongestEntryLength(FeatTable);
316
317  printf("Available features for this target:\n");
318  for (auto &Feature : FeatTable)
319    printf("    %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc);
320  printf("\n");
321
322  printf("Use +feature to enable a feature, or -feature to disable it.\n"
323         "For example, rustc -C -target-cpu=mycpu -C "
324         "target-feature=+feature1,-feature2\n\n");
325}
326
327#else
328
329extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
330  printf("Target CPU help is not supported by this LLVM version.\n\n");
331}
332
333extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) {
334  printf("Target features help is not supported by this LLVM version.\n\n");
335}
336#endif
337
338extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
339  StringRef Name = sys::getHostCPUName();
340  *len = Name.size();
341  return Name.data();
342}
343
344extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
345    const char *TripleStr, const char *CPU, const char *Feature,
346    LLVMRustCodeModel RustCM, LLVMRustRelocMode RustReloc,
347    LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
348    bool PositionIndependentExecutable, bool FunctionSections,
349    bool DataSections,
350    bool TrapUnreachable,
351    bool Singlethread,
352    bool AsmComments,
353    bool EmitStackSizeSection) {
354
355  auto OptLevel = fromRust(RustOptLevel);
356  auto RM = fromRust(RustReloc);
357
358  std::string Error;
359  Triple Trip(Triple::normalize(TripleStr));
360  const llvm::Target *TheTarget =
361      TargetRegistry::lookupTarget(Trip.getTriple(), Error);
362  if (TheTarget == nullptr) {
363    LLVMRustSetLastError(Error.c_str());
364    return nullptr;
365  }
366
367  TargetOptions Options;
368
369  Options.FloatABIType = FloatABI::Default;
370  if (UseSoftFloat) {
371    Options.FloatABIType = FloatABI::Soft;
372  }
373  Options.DataSections = DataSections;
374  Options.FunctionSections = FunctionSections;
375  Options.MCOptions.AsmVerbose = AsmComments;
376  Options.MCOptions.PreserveAsmComments = AsmComments;
377
378  if (TrapUnreachable) {
379    // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
380    // This limits the extent of possible undefined behavior in some cases, as
381    // it prevents control flow from "falling through" into whatever code
382    // happens to be laid out next in memory.
383    Options.TrapUnreachable = true;
384  }
385
386  if (Singlethread) {
387    Options.ThreadModel = ThreadModel::Single;
388  }
389
390  Options.EmitStackSizeSection = EmitStackSizeSection;
391
392  Optional<CodeModel::Model> CM;
393  if (RustCM != LLVMRustCodeModel::None)
394    CM = fromRust(RustCM);
395  TargetMachine *TM = TheTarget->createTargetMachine(
396      Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
397  return wrap(TM);
398}
399
400extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
401  delete unwrap(TM);
402}
403
404// Unfortunately, LLVM doesn't expose a C API to add the corresponding analysis
405// passes for a target to a pass manager. We export that functionality through
406// this function.
407extern "C" void LLVMRustAddAnalysisPasses(LLVMTargetMachineRef TM,
408                                          LLVMPassManagerRef PMR,
409                                          LLVMModuleRef M) {
410  PassManagerBase *PM = unwrap(PMR);
411  PM->add(
412      createTargetTransformInfoWrapperPass(unwrap(TM)->getTargetIRAnalysis()));
413}
414
415extern "C" void LLVMRustConfigurePassManagerBuilder(
416    LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
417    bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
418    const char* PGOGenPath, const char* PGOUsePath) {
419#if LLVM_VERSION_GE(7, 0)
420  unwrap(PMBR)->MergeFunctions = MergeFunctions;
421#endif
422  unwrap(PMBR)->SLPVectorize = SLPVectorize;
423  unwrap(PMBR)->OptLevel = fromRust(OptLevel);
424  unwrap(PMBR)->LoopVectorize = LoopVectorize;
425  unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
426
427  if (PGOGenPath) {
428    assert(!PGOUsePath);
429    unwrap(PMBR)->EnablePGOInstrGen = true;
430    unwrap(PMBR)->PGOInstrGen = PGOGenPath;
431  }
432  if (PGOUsePath) {
433    assert(!PGOGenPath);
434    unwrap(PMBR)->PGOInstrUse = PGOUsePath;
435  }
436}
437
438// Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
439// field of a PassManagerBuilder, we expose our own method of doing so.
440extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
441                                              LLVMModuleRef M,
442                                              bool DisableSimplifyLibCalls) {
443  Triple TargetTriple(unwrap(M)->getTargetTriple());
444  TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
445  if (DisableSimplifyLibCalls)
446    TLI->disableAllFunctions();
447  unwrap(PMBR)->LibraryInfo = TLI;
448}
449
450// Unfortunately, the LLVM C API doesn't provide a way to create the
451// TargetLibraryInfo pass, so we use this method to do so.
452extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
453                                       bool DisableSimplifyLibCalls) {
454  Triple TargetTriple(unwrap(M)->getTargetTriple());
455  TargetLibraryInfoImpl TLII(TargetTriple);
456  if (DisableSimplifyLibCalls)
457    TLII.disableAllFunctions();
458  unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
459}
460
461// Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
462// all the functions in a module, so we do that manually here. You'll find
463// similar code in clang's BackendUtil.cpp file.
464extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
465                                               LLVMModuleRef M) {
466  llvm::legacy::FunctionPassManager *P =
467      unwrap<llvm::legacy::FunctionPassManager>(PMR);
468  P->doInitialization();
469
470  // Upgrade all calls to old intrinsics first.
471  for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
472    UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
473
474  for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
475       ++I)
476    if (!I->isDeclaration())
477      P->run(*I);
478
479  P->doFinalization();
480}
481
482extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
483  // Initializing the command-line options more than once is not allowed. So,
484  // check if they've already been initialized.  (This could happen if we're
485  // being called from rustpkg, for example). If the arguments change, then
486  // that's just kinda unfortunate.
487  static bool Initialized = false;
488  if (Initialized)
489    return;
490  Initialized = true;
491  cl::ParseCommandLineOptions(Argc, Argv);
492}
493
494enum class LLVMRustFileType {
495  Other,
496  AssemblyFile,
497  ObjectFile,
498};
499
500static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) {
501  switch (Type) {
502  case LLVMRustFileType::AssemblyFile:
503    return TargetMachine::CGFT_AssemblyFile;
504  case LLVMRustFileType::ObjectFile:
505    return TargetMachine::CGFT_ObjectFile;
506  default:
507    report_fatal_error("Bad FileType.");
508  }
509}
510
511extern "C" LLVMRustResult
512LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
513                        LLVMModuleRef M, const char *Path,
514                        LLVMRustFileType RustFileType) {
515  llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
516  auto FileType = fromRust(RustFileType);
517
518  std::string ErrorInfo;
519  std::error_code EC;
520  raw_fd_ostream OS(Path, EC, sys::fs::F_None);
521  if (EC)
522    ErrorInfo = EC.message();
523  if (ErrorInfo != "") {
524    LLVMRustSetLastError(ErrorInfo.c_str());
525    return LLVMRustResult::Failure;
526  }
527
528#if LLVM_VERSION_GE(7, 0)
529  buffer_ostream BOS(OS);
530  unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
531#else
532  unwrap(Target)->addPassesToEmitFile(*PM, OS, FileType, false);
533#endif
534  PM->run(*unwrap(M));
535
536  // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
537  // stream (OS), so the only real safe place to delete this is here? Don't we
538  // wish this was written in Rust?
539  delete PM;
540  return LLVMRustResult::Success;
541}
542
543
544// Callback to demangle function name
545// Parameters:
546// * name to be demangled
547// * name len
548// * output buffer
549// * output buffer len
550// Returns len of demangled string, or 0 if demangle failed.
551typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
552
553
554namespace {
555
556class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
557  DemangleFn Demangle;
558  std::vector<char> Buf;
559
560public:
561  RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
562
563  // Return empty string if demangle failed
564  // or if name does not need to be demangled
565  StringRef CallDemangle(StringRef name) {
566    if (!Demangle) {
567      return StringRef();
568    }
569
570    if (Buf.size() < name.size() * 2) {
571      // Semangled name usually shorter than mangled,
572      // but allocate twice as much memory just in case
573      Buf.resize(name.size() * 2);
574    }
575
576    auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
577    if (!R) {
578      // Demangle failed.
579      return StringRef();
580    }
581
582    auto Demangled = StringRef(Buf.data(), R);
583    if (Demangled == name) {
584      // Do not print anything if demangled name is equal to mangled.
585      return StringRef();
586    }
587
588    return Demangled;
589  }
590
591  void emitFunctionAnnot(const Function *F,
592                         formatted_raw_ostream &OS) override {
593    StringRef Demangled = CallDemangle(F->getName());
594    if (Demangled.empty()) {
595        return;
596    }
597
598    OS << "; " << Demangled << "\n";
599  }
600
601  void emitInstructionAnnot(const Instruction *I,
602                            formatted_raw_ostream &OS) override {
603    const char *Name;
604    const Value *Value;
605    if (const CallInst *CI = dyn_cast<CallInst>(I)) {
606      Name = "call";
607      Value = CI->getCalledValue();
608    } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
609      Name = "invoke";
610      Value = II->getCalledValue();
611    } else {
612      // Could demangle more operations, e. g.
613      // `store %place, @function`.
614      return;
615    }
616
617    if (!Value->hasName()) {
618      return;
619    }
620
621    StringRef Demangled = CallDemangle(Value->getName());
622    if (Demangled.empty()) {
623      return;
624    }
625
626    OS << "; " << Name << " " << Demangled << "\n";
627  }
628};
629
630class RustPrintModulePass : public ModulePass {
631  raw_ostream* OS;
632  DemangleFn Demangle;
633public:
634  static char ID;
635  RustPrintModulePass() : ModulePass(ID), OS(nullptr), Demangle(nullptr) {}
636  RustPrintModulePass(raw_ostream &OS, DemangleFn Demangle)
637      : ModulePass(ID), OS(&OS), Demangle(Demangle) {}
638
639  bool runOnModule(Module &M) override {
640    RustAssemblyAnnotationWriter AW(Demangle);
641
642    M.print(*OS, &AW, false);
643
644    return false;
645  }
646
647  void getAnalysisUsage(AnalysisUsage &AU) const override {
648    AU.setPreservesAll();
649  }
650
651  static StringRef name() { return "RustPrintModulePass"; }
652};
653
654} // namespace
655
656namespace llvm {
657  void initializeRustPrintModulePassPass(PassRegistry&);
658}
659
660char RustPrintModulePass::ID = 0;
661INITIALIZE_PASS(RustPrintModulePass, "print-rust-module",
662                "Print rust module to stderr", false, false)
663
664extern "C" LLVMRustResult
665LLVMRustPrintModule(LLVMPassManagerRef PMR, LLVMModuleRef M,
666                    const char *Path, DemangleFn Demangle) {
667  llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
668  std::string ErrorInfo;
669
670  std::error_code EC;
671  raw_fd_ostream OS(Path, EC, sys::fs::F_None);
672  if (EC)
673    ErrorInfo = EC.message();
674  if (ErrorInfo != "") {
675    LLVMRustSetLastError(ErrorInfo.c_str());
676    return LLVMRustResult::Failure;
677  }
678
679  formatted_raw_ostream FOS(OS);
680
681  PM->add(new RustPrintModulePass(FOS, Demangle));
682
683  PM->run(*unwrap(M));
684
685  return LLVMRustResult::Success;
686}
687
688extern "C" void LLVMRustPrintPasses() {
689  LLVMInitializePasses();
690  struct MyListener : PassRegistrationListener {
691    void passEnumerate(const PassInfo *Info) {
692      StringRef PassArg = Info->getPassArgument();
693      StringRef PassName = Info->getPassName();
694      if (!PassArg.empty()) {
695        // These unsigned->signed casts could theoretically overflow, but
696        // realistically never will (and even if, the result is implementation
697        // defined rather plain UB).
698        printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
699               (int)PassName.size(), PassName.data());
700      }
701    }
702  } Listener;
703
704  PassRegistry *PR = PassRegistry::getPassRegistry();
705  PR->enumerateWith(&Listener);
706}
707
708extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
709                                            bool AddLifetimes) {
710  unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
711}
712
713extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
714                                           size_t Len) {
715  llvm::legacy::PassManager passes;
716
717  auto PreserveFunctions = [=](const GlobalValue &GV) {
718    for (size_t I = 0; I < Len; I++) {
719      if (GV.getName() == Symbols[I]) {
720        return true;
721      }
722    }
723    return false;
724  };
725
726  passes.add(llvm::createInternalizePass(PreserveFunctions));
727
728  passes.run(*unwrap(M));
729}
730
731extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) {
732  for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E;
733       ++GV) {
734    GV->setDoesNotThrow();
735    Function *F = dyn_cast<Function>(GV);
736    if (F == nullptr)
737      continue;
738
739    for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
740      for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) {
741        if (isa<InvokeInst>(I)) {
742          InvokeInst *CI = cast<InvokeInst>(I);
743          CI->setDoesNotThrow();
744        }
745      }
746    }
747  }
748}
749
750extern "C" void
751LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
752                                       LLVMTargetMachineRef TMR) {
753  TargetMachine *Target = unwrap(TMR);
754  unwrap(Module)->setDataLayout(Target->createDataLayout());
755}
756
757extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
758  unwrap(M)->setPIELevel(PIELevel::Level::Large);
759}
760
761// Here you'll find an implementation of ThinLTO as used by the Rust compiler
762// right now. This ThinLTO support is only enabled on "recent ish" versions of
763// LLVM, and otherwise it's just blanket rejected from other compilers.
764//
765// Most of this implementation is straight copied from LLVM. At the time of
766// this writing it wasn't *quite* suitable to reuse more code from upstream
767// for our purposes, but we should strive to upstream this support once it's
768// ready to go! I figure we may want a bit of testing locally first before
769// sending this upstream to LLVM. I hear though they're quite eager to receive
770// feedback like this!
771//
772// If you're reading this code and wondering "what in the world" or you're
773// working "good lord by LLVM upgrade is *still* failing due to these bindings"
774// then fear not! (ok maybe fear a little). All code here is mostly based
775// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
776//
777// You'll find that the general layout here roughly corresponds to the `run`
778// method in that file as well as `ProcessThinLTOModule`. Functions are
779// specifically commented below as well, but if you're updating this code
780// or otherwise trying to understand it, the LLVM source will be useful in
781// interpreting the mysteries within.
782//
783// Otherwise I'll apologize in advance, it probably requires a relatively
784// significant investment on your part to "truly understand" what's going on
785// here. Not saying I do myself, but it took me awhile staring at LLVM's source
786// and various online resources about ThinLTO to make heads or tails of all
787// this.
788
789// This is a shared data structure which *must* be threadsafe to share
790// read-only amongst threads. This also corresponds basically to the arguments
791// of the `ProcessThinLTOModule` function in the LLVM source.
792struct LLVMRustThinLTOData {
793  // The combined index that is the global analysis over all modules we're
794  // performing ThinLTO for. This is mostly managed by LLVM.
795  ModuleSummaryIndex Index;
796
797  // All modules we may look at, stored as in-memory serialized versions. This
798  // is later used when inlining to ensure we can extract any module to inline
799  // from.
800  StringMap<MemoryBufferRef> ModuleMap;
801
802  // A set that we manage of everything we *don't* want internalized. Note that
803  // this includes all transitive references right now as well, but it may not
804  // always!
805  DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
806
807  // Not 100% sure what these are, but they impact what's internalized and
808  // what's inlined across modules, I believe.
809  StringMap<FunctionImporter::ImportMapTy> ImportLists;
810  StringMap<FunctionImporter::ExportSetTy> ExportLists;
811  StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
812
813#if LLVM_VERSION_GE(7, 0)
814  LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
815#endif
816};
817
818// Just an argument to the `LLVMRustCreateThinLTOData` function below.
819struct LLVMRustThinLTOModule {
820  const char *identifier;
821  const char *data;
822  size_t len;
823};
824
825// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
826// does.
827static const GlobalValueSummary *
828getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
829  auto StrongDefForLinker = llvm::find_if(
830      GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
831        auto Linkage = Summary->linkage();
832        return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
833               !GlobalValue::isWeakForLinker(Linkage);
834      });
835  if (StrongDefForLinker != GVSummaryList.end())
836    return StrongDefForLinker->get();
837
838  auto FirstDefForLinker = llvm::find_if(
839      GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
840        auto Linkage = Summary->linkage();
841        return !GlobalValue::isAvailableExternallyLinkage(Linkage);
842      });
843  if (FirstDefForLinker == GVSummaryList.end())
844    return nullptr;
845  return FirstDefForLinker->get();
846}
847
848// The main entry point for creating the global ThinLTO analysis. The structure
849// here is basically the same as before threads are spawned in the `run`
850// function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
851extern "C" LLVMRustThinLTOData*
852LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
853                          int num_modules,
854                          const char **preserved_symbols,
855                          int num_symbols) {
856  auto Ret = llvm::make_unique<LLVMRustThinLTOData>();
857
858  // Load each module's summary and merge it into one combined index
859  for (int i = 0; i < num_modules; i++) {
860    auto module = &modules[i];
861    StringRef buffer(module->data, module->len);
862    MemoryBufferRef mem_buffer(buffer, module->identifier);
863
864    Ret->ModuleMap[module->identifier] = mem_buffer;
865
866    if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
867      LLVMRustSetLastError(toString(std::move(Err)).c_str());
868      return nullptr;
869    }
870  }
871
872  // Collect for each module the list of function it defines (GUID -> Summary)
873  Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
874
875  // Convert the preserved symbols set from string to GUID, this is then needed
876  // for internalization.
877  for (int i = 0; i < num_symbols; i++) {
878    auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
879    Ret->GUIDPreservedSymbols.insert(GUID);
880  }
881
882  // Collect the import/export lists for all modules from the call-graph in the
883  // combined index
884  //
885  // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
886#if LLVM_VERSION_GE(7, 0)
887  auto deadIsPrevailing = [&](GlobalValue::GUID G) {
888    return PrevailingType::Unknown;
889  };
890#if LLVM_VERSION_GE(8, 0)
891  // We don't have a complete picture in our use of ThinLTO, just our immediate
892  // crate, so we need `ImportEnabled = false` to limit internalization.
893  // Otherwise, we sometimes lose `static` values -- see #60184.
894  computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
895                                  deadIsPrevailing, /* ImportEnabled = */ false);
896#else
897  computeDeadSymbols(Ret->Index, Ret->GUIDPreservedSymbols, deadIsPrevailing);
898#endif
899#else
900  computeDeadSymbols(Ret->Index, Ret->GUIDPreservedSymbols);
901#endif
902  ComputeCrossModuleImport(
903    Ret->Index,
904    Ret->ModuleToDefinedGVSummaries,
905    Ret->ImportLists,
906    Ret->ExportLists
907  );
908
909  // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
910  // impacts the caching.
911  //
912  // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
913  // being lifted from `lib/LTO/LTO.cpp` as well
914  StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
915  DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
916  for (auto &I : Ret->Index) {
917    if (I.second.SummaryList.size() > 1)
918      PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
919  }
920  auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
921    const auto &Prevailing = PrevailingCopy.find(GUID);
922    if (Prevailing == PrevailingCopy.end())
923      return true;
924    return Prevailing->second == S;
925  };
926  auto recordNewLinkage = [&](StringRef ModuleIdentifier,
927                              GlobalValue::GUID GUID,
928                              GlobalValue::LinkageTypes NewLinkage) {
929    ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
930  };
931#if LLVM_VERSION_GE(9, 0)
932  thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage,
933                                  Ret->GUIDPreservedSymbols);
934#elif LLVM_VERSION_GE(8, 0)
935  thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage);
936#else
937  thinLTOResolveWeakForLinkerInIndex(Ret->Index, isPrevailing, recordNewLinkage);
938#endif
939
940  // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
941  // callback below. This callback below will dictate the linkage for all
942  // summaries in the index, and we basically just only want to ensure that dead
943  // symbols are internalized. Otherwise everything that's already external
944  // linkage will stay as external, and internal will stay as internal.
945  std::set<GlobalValue::GUID> ExportedGUIDs;
946  for (auto &List : Ret->Index) {
947    for (auto &GVS: List.second.SummaryList) {
948      if (GlobalValue::isLocalLinkage(GVS->linkage()))
949        continue;
950      auto GUID = GVS->getOriginalName();
951      if (GVS->flags().Live)
952        ExportedGUIDs.insert(GUID);
953    }
954  }
955  auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
956    const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
957    return (ExportList != Ret->ExportLists.end() &&
958      ExportList->second.count(GUID)) ||
959      ExportedGUIDs.count(GUID);
960  };
961  thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported);
962
963  return Ret.release();
964}
965
966extern "C" void
967LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
968  delete Data;
969}
970
971// Below are the various passes that happen *per module* when doing ThinLTO.
972//
973// In other words, these are the functions that are all run concurrently
974// with one another, one per module. The passes here correspond to the analysis
975// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
976// `ProcessThinLTOModule` function. Here they're split up into separate steps
977// so rustc can save off the intermediate bytecode between each step.
978
979extern "C" bool
980LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
981  Module &Mod = *unwrap(M);
982  if (renameModuleForThinLTO(Mod, Data->Index)) {
983    LLVMRustSetLastError("renameModuleForThinLTO failed");
984    return false;
985  }
986  return true;
987}
988
989extern "C" bool
990LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
991  Module &Mod = *unwrap(M);
992  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
993#if LLVM_VERSION_GE(8, 0)
994  thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
995#else
996  thinLTOResolveWeakForLinkerModule(Mod, DefinedGlobals);
997#endif
998  return true;
999}
1000
1001extern "C" bool
1002LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1003  Module &Mod = *unwrap(M);
1004  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1005  thinLTOInternalizeModule(Mod, DefinedGlobals);
1006  return true;
1007}
1008
1009extern "C" bool
1010LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1011  Module &Mod = *unwrap(M);
1012
1013  const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1014  auto Loader = [&](StringRef Identifier) {
1015    const auto &Memory = Data->ModuleMap.lookup(Identifier);
1016    auto &Context = Mod.getContext();
1017    auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1018
1019    if (!MOrErr)
1020      return MOrErr;
1021
1022    // The rest of this closure is a workaround for
1023    // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1024    // we accidentally import wasm custom sections into different modules,
1025    // duplicating them by in the final output artifact.
1026    //
1027    // The issue is worked around here by manually removing the
1028    // `wasm.custom_sections` named metadata node from any imported module. This
1029    // we know isn't used by any optimization pass so there's no need for it to
1030    // be imported.
1031    //
1032    // Note that the metadata is currently lazily loaded, so we materialize it
1033    // here before looking up if there's metadata inside. The `FunctionImporter`
1034    // will immediately materialize metadata anyway after an import, so this
1035    // shouldn't be a perf hit.
1036    if (Error Err = (*MOrErr)->materializeMetadata()) {
1037      Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1038      return Ret;
1039    }
1040
1041    auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1042    if (WasmCustomSections)
1043      WasmCustomSections->eraseFromParent();
1044
1045    return MOrErr;
1046  };
1047  FunctionImporter Importer(Data->Index, Loader);
1048  Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1049  if (!Result) {
1050    LLVMRustSetLastError(toString(Result.takeError()).c_str());
1051    return false;
1052  }
1053  return true;
1054}
1055
1056extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload
1057                                                      const char*, // importing module name
1058                                                      const char*); // imported module name
1059
1060// Calls `module_name_callback` for each module import done by ThinLTO.
1061// The callback is provided with regular null-terminated C strings.
1062extern "C" void
1063LLVMRustGetThinLTOModuleImports(const LLVMRustThinLTOData *data,
1064                                LLVMRustModuleNameCallback module_name_callback,
1065                                void* callback_payload) {
1066  for (const auto& importing_module : data->ImportLists) {
1067    const std::string importing_module_id = importing_module.getKey().str();
1068    const auto& imports = importing_module.getValue();
1069    for (const auto& imported_module : imports) {
1070      const std::string imported_module_id = imported_module.getKey().str();
1071      module_name_callback(callback_payload,
1072                           importing_module_id.c_str(),
1073                           imported_module_id.c_str());
1074    }
1075  }
1076}
1077
1078// This struct and various functions are sort of a hack right now, but the
1079// problem is that we've got in-memory LLVM modules after we generate and
1080// optimize all codegen-units for one compilation in rustc. To be compatible
1081// with the LTO support above we need to serialize the modules plus their
1082// ThinLTO summary into memory.
1083//
1084// This structure is basically an owned version of a serialize module, with
1085// a ThinLTO summary attached.
1086struct LLVMRustThinLTOBuffer {
1087  std::string data;
1088};
1089
1090extern "C" LLVMRustThinLTOBuffer*
1091LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
1092  auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>();
1093  {
1094    raw_string_ostream OS(Ret->data);
1095    {
1096      legacy::PassManager PM;
1097      PM.add(createWriteThinLTOBitcodePass(OS));
1098      PM.run(*unwrap(M));
1099    }
1100  }
1101  return Ret.release();
1102}
1103
1104extern "C" void
1105LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1106  delete Buffer;
1107}
1108
1109extern "C" const void*
1110LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1111  return Buffer->data.data();
1112}
1113
1114extern "C" size_t
1115LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1116  return Buffer->data.length();
1117}
1118
1119// This is what we used to parse upstream bitcode for actual ThinLTO
1120// processing.  We'll call this once per module optimized through ThinLTO, and
1121// it'll be called concurrently on many threads.
1122extern "C" LLVMModuleRef
1123LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1124                           const char *data,
1125                           size_t len,
1126                           const char *identifier) {
1127  StringRef Data(data, len);
1128  MemoryBufferRef Buffer(Data, identifier);
1129  unwrap(Context)->enableDebugTypeODRUniquing();
1130  Expected<std::unique_ptr<Module>> SrcOrError =
1131      parseBitcodeFile(Buffer, *unwrap(Context));
1132  if (!SrcOrError) {
1133    LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1134    return nullptr;
1135  }
1136  return wrap(std::move(*SrcOrError).release());
1137}
1138
1139// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1140// the comment in `back/lto.rs` for why this exists.
1141extern "C" void
1142LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1143                                DICompileUnit **A,
1144                                DICompileUnit **B) {
1145  Module *M = unwrap(Mod);
1146  DICompileUnit **Cur = A;
1147  DICompileUnit **Next = B;
1148  for (DICompileUnit *CU : M->debug_compile_units()) {
1149    *Cur = CU;
1150    Cur = Next;
1151    Next = nullptr;
1152    if (Cur == nullptr)
1153      break;
1154  }
1155}
1156
1157// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1158// the comment in `back/lto.rs` for why this exists.
1159extern "C" void
1160LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1161  Module *M = unwrap(Mod);
1162
1163  // If the original source module didn't have a `DICompileUnit` then try to
1164  // merge all the existing compile units. If there aren't actually any though
1165  // then there's not much for us to do so return.
1166  if (Unit == nullptr) {
1167    for (DICompileUnit *CU : M->debug_compile_units()) {
1168      Unit = CU;
1169      break;
1170    }
1171    if (Unit == nullptr)
1172      return;
1173  }
1174
1175  // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1176  // process it recursively. Note that we specifically iterate over instructions
1177  // to ensure we feed everything into it.
1178  DebugInfoFinder Finder;
1179  Finder.processModule(*M);
1180  for (Function &F : M->functions()) {
1181    for (auto &FI : F) {
1182      for (Instruction &BI : FI) {
1183        if (auto Loc = BI.getDebugLoc())
1184          Finder.processLocation(*M, Loc);
1185        if (auto DVI = dyn_cast<DbgValueInst>(&BI))
1186          Finder.processValue(*M, DVI);
1187        if (auto DDI = dyn_cast<DbgDeclareInst>(&BI))
1188          Finder.processDeclare(*M, DDI);
1189      }
1190    }
1191  }
1192
1193  // After we've found all our debuginfo, rewrite all subprograms to point to
1194  // the same `DICompileUnit`.
1195  for (auto &F : Finder.subprograms()) {
1196    F->replaceUnit(Unit);
1197  }
1198
1199  // Erase any other references to other `DICompileUnit` instances, the verifier
1200  // will later ensure that we don't actually have any other stale references to
1201  // worry about.
1202  auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1203  MD->clearOperands();
1204  MD->addOperand(Unit);
1205}