fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #69854 - pietroalbini:stable-next, r=Centrilb8cedc00407

1.42.0
40.3 KiB1229 linesraw
1#include <stdio.h>
2
3#include <vector>
4#include <set>
5
6#include "rustllvm.h"
7
8#include "llvm/Analysis/TargetLibraryInfo.h"
9#include "llvm/Analysis/TargetTransformInfo.h"
10#include "llvm/CodeGen/TargetSubtargetInfo.h"
11#include "llvm/InitializePasses.h"
12#include "llvm/IR/AutoUpgrade.h"
13#include "llvm/IR/AssemblyAnnotationWriter.h"
14#include "llvm/IR/IntrinsicInst.h"
15#include "llvm/Support/CBindingWrapping.h"
16#include "llvm/Support/FileSystem.h"
17#include "llvm/Support/Host.h"
18#include "llvm/Target/TargetMachine.h"
19#include "llvm/Transforms/IPO/PassManagerBuilder.h"
20#include "llvm/Transforms/IPO/AlwaysInliner.h"
21#include "llvm/Transforms/IPO/FunctionImport.h"
22#include "llvm/Transforms/Utils/FunctionImportUtils.h"
23#include "llvm/LTO/LTO.h"
24#include "llvm-c/Transforms/PassManagerBuilder.h"
25
26#include "llvm/Transforms/Instrumentation.h"
27#if LLVM_VERSION_GE(9, 0)
28#include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
29#endif
30#if LLVM_VERSION_GE(8, 0)
31#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
32#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
33#endif
34
35using namespace llvm;
36using namespace llvm::legacy;
37
38typedef struct LLVMOpaquePass *LLVMPassRef;
39typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
40
41DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
42DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
43DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder,
44                                   LLVMPassManagerBuilderRef)
45
46extern "C" void LLVMInitializePasses() {
47  PassRegistry &Registry = *PassRegistry::getPassRegistry();
48  initializeCore(Registry);
49  initializeCodeGen(Registry);
50  initializeScalarOpts(Registry);
51  initializeVectorization(Registry);
52  initializeIPO(Registry);
53  initializeAnalysis(Registry);
54  initializeTransformUtils(Registry);
55  initializeInstCombine(Registry);
56  initializeInstrumentation(Registry);
57  initializeTarget(Registry);
58}
59
60enum class LLVMRustPassKind {
61  Other,
62  Function,
63  Module,
64};
65
66static LLVMRustPassKind toRust(PassKind Kind) {
67  switch (Kind) {
68  case PT_Function:
69    return LLVMRustPassKind::Function;
70  case PT_Module:
71    return LLVMRustPassKind::Module;
72  default:
73    return LLVMRustPassKind::Other;
74  }
75}
76
77extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) {
78  StringRef SR(PassName);
79  PassRegistry *PR = PassRegistry::getPassRegistry();
80
81  const PassInfo *PI = PR->getPassInfo(SR);
82  if (PI) {
83    return wrap(PI->createPass());
84  }
85  return nullptr;
86}
87
88extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) {
89  const bool CompileKernel = false;
90
91  return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover));
92}
93
94extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) {
95  const bool CompileKernel = false;
96
97#if LLVM_VERSION_GE(9, 0)
98  return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover));
99#else
100  return wrap(createAddressSanitizerModulePass(CompileKernel, Recover));
101#endif
102}
103
104extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) {
105#if LLVM_VERSION_GE(9, 0)
106  const bool CompileKernel = false;
107
108  return wrap(createMemorySanitizerLegacyPassPass(
109      MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel}));
110#elif LLVM_VERSION_GE(8, 0)
111  return wrap(createMemorySanitizerLegacyPassPass(TrackOrigins, Recover));
112#else
113  return wrap(createMemorySanitizerPass(TrackOrigins, Recover));
114#endif
115}
116
117extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() {
118#if LLVM_VERSION_GE(8, 0)
119  return wrap(createThreadSanitizerLegacyPassPass());
120#else
121  return wrap(createThreadSanitizerPass());
122#endif
123}
124
125extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) {
126  assert(RustPass);
127  Pass *Pass = unwrap(RustPass);
128  return toRust(Pass->getPassKind());
129}
130
131extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) {
132  assert(RustPass);
133  Pass *Pass = unwrap(RustPass);
134  PassManagerBase *PMB = unwrap(PMR);
135  PMB->add(Pass);
136}
137
138extern "C"
139void LLVMRustPassManagerBuilderPopulateThinLTOPassManager(
140  LLVMPassManagerBuilderRef PMBR,
141  LLVMPassManagerRef PMR
142) {
143  unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR));
144}
145
146extern "C"
147void LLVMRustAddLastExtensionPasses(
148    LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) {
149  auto AddExtensionPasses = [Passes, NumPasses](
150      const PassManagerBuilder &Builder, PassManagerBase &PM) {
151    for (size_t I = 0; I < NumPasses; I++) {
152      PM.add(unwrap(Passes[I]));
153    }
154  };
155  // Add the passes to both of the pre-finalization extension points,
156  // so they are run for optimized and non-optimized builds.
157  unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast,
158                             AddExtensionPasses);
159  unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
160                             AddExtensionPasses);
161}
162
163#ifdef LLVM_COMPONENT_X86
164#define SUBTARGET_X86 SUBTARGET(X86)
165#else
166#define SUBTARGET_X86
167#endif
168
169#ifdef LLVM_COMPONENT_ARM
170#define SUBTARGET_ARM SUBTARGET(ARM)
171#else
172#define SUBTARGET_ARM
173#endif
174
175#ifdef LLVM_COMPONENT_AARCH64
176#define SUBTARGET_AARCH64 SUBTARGET(AArch64)
177#else
178#define SUBTARGET_AARCH64
179#endif
180
181#ifdef LLVM_COMPONENT_MIPS
182#define SUBTARGET_MIPS SUBTARGET(Mips)
183#else
184#define SUBTARGET_MIPS
185#endif
186
187#ifdef LLVM_COMPONENT_POWERPC
188#define SUBTARGET_PPC SUBTARGET(PPC)
189#else
190#define SUBTARGET_PPC
191#endif
192
193#ifdef LLVM_COMPONENT_SYSTEMZ
194#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
195#else
196#define SUBTARGET_SYSTEMZ
197#endif
198
199#ifdef LLVM_COMPONENT_MSP430
200#define SUBTARGET_MSP430 SUBTARGET(MSP430)
201#else
202#define SUBTARGET_MSP430
203#endif
204
205#ifdef LLVM_COMPONENT_RISCV
206#define SUBTARGET_RISCV SUBTARGET(RISCV)
207#else
208#define SUBTARGET_RISCV
209#endif
210
211#ifdef LLVM_COMPONENT_SPARC
212#define SUBTARGET_SPARC SUBTARGET(Sparc)
213#else
214#define SUBTARGET_SPARC
215#endif
216
217#ifdef LLVM_COMPONENT_HEXAGON
218#define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
219#else
220#define SUBTARGET_HEXAGON
221#endif
222
223#define GEN_SUBTARGETS                                                         \
224  SUBTARGET_X86                                                                \
225  SUBTARGET_ARM                                                                \
226  SUBTARGET_AARCH64                                                            \
227  SUBTARGET_MIPS                                                               \
228  SUBTARGET_PPC                                                                \
229  SUBTARGET_SYSTEMZ                                                            \
230  SUBTARGET_MSP430                                                             \
231  SUBTARGET_SPARC                                                              \
232  SUBTARGET_HEXAGON                                                            \
233  SUBTARGET_RISCV                                                              \
234
235#define SUBTARGET(x)                                                           \
236  namespace llvm {                                                             \
237  extern const SubtargetFeatureKV x##FeatureKV[];                              \
238  extern const SubtargetFeatureKV x##SubTypeKV[];                              \
239  }
240
241GEN_SUBTARGETS
242#undef SUBTARGET
243
244extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
245                                   const char *Feature) {
246  TargetMachine *Target = unwrap(TM);
247  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
248  return MCInfo->checkFeatures(std::string("+") + Feature);
249}
250
251enum class LLVMRustCodeModel {
252  Other,
253  Small,
254  Kernel,
255  Medium,
256  Large,
257  None,
258};
259
260static CodeModel::Model fromRust(LLVMRustCodeModel Model) {
261  switch (Model) {
262  case LLVMRustCodeModel::Small:
263    return CodeModel::Small;
264  case LLVMRustCodeModel::Kernel:
265    return CodeModel::Kernel;
266  case LLVMRustCodeModel::Medium:
267    return CodeModel::Medium;
268  case LLVMRustCodeModel::Large:
269    return CodeModel::Large;
270  default:
271    report_fatal_error("Bad CodeModel.");
272  }
273}
274
275enum class LLVMRustCodeGenOptLevel {
276  Other,
277  None,
278  Less,
279  Default,
280  Aggressive,
281};
282
283static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
284  switch (Level) {
285  case LLVMRustCodeGenOptLevel::None:
286    return CodeGenOpt::None;
287  case LLVMRustCodeGenOptLevel::Less:
288    return CodeGenOpt::Less;
289  case LLVMRustCodeGenOptLevel::Default:
290    return CodeGenOpt::Default;
291  case LLVMRustCodeGenOptLevel::Aggressive:
292    return CodeGenOpt::Aggressive;
293  default:
294    report_fatal_error("Bad CodeGenOptLevel.");
295  }
296}
297
298enum class LLVMRustRelocMode {
299  Default,
300  Static,
301  PIC,
302  DynamicNoPic,
303  ROPI,
304  RWPI,
305  ROPIRWPI,
306};
307
308static Optional<Reloc::Model> fromRust(LLVMRustRelocMode RustReloc) {
309  switch (RustReloc) {
310  case LLVMRustRelocMode::Default:
311    return None;
312  case LLVMRustRelocMode::Static:
313    return Reloc::Static;
314  case LLVMRustRelocMode::PIC:
315    return Reloc::PIC_;
316  case LLVMRustRelocMode::DynamicNoPic:
317    return Reloc::DynamicNoPIC;
318  case LLVMRustRelocMode::ROPI:
319    return Reloc::ROPI;
320  case LLVMRustRelocMode::RWPI:
321    return Reloc::RWPI;
322  case LLVMRustRelocMode::ROPIRWPI:
323    return Reloc::ROPI_RWPI;
324  }
325  report_fatal_error("Bad RelocModel.");
326}
327
328#ifdef LLVM_RUSTLLVM
329/// getLongestEntryLength - Return the length of the longest entry in the table.
330template<typename KV>
331static size_t getLongestEntryLength(ArrayRef<KV> Table) {
332  size_t MaxLen = 0;
333  for (auto &I : Table)
334    MaxLen = std::max(MaxLen, std::strlen(I.Key));
335  return MaxLen;
336}
337
338extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) {
339  const TargetMachine *Target = unwrap(TM);
340  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
341  const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch();
342  const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
343  const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable();
344  unsigned MaxCPULen = getLongestEntryLength(CPUTable);
345
346  printf("Available CPUs for this target:\n");
347  if (HostArch == TargetArch) {
348    const StringRef HostCPU = sys::getHostCPUName();
349    printf("    %-*s - Select the CPU of the current host (currently %.*s).\n",
350      MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data());
351  }
352  for (auto &CPU : CPUTable)
353    printf("    %-*s\n", MaxCPULen, CPU.Key);
354  printf("\n");
355}
356
357extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) {
358  const TargetMachine *Target = unwrap(TM);
359  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
360  const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
361  unsigned MaxFeatLen = getLongestEntryLength(FeatTable);
362
363  printf("Available features for this target:\n");
364  for (auto &Feature : FeatTable)
365    printf("    %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc);
366  printf("\n");
367
368  printf("Use +feature to enable a feature, or -feature to disable it.\n"
369         "For example, rustc -C -target-cpu=mycpu -C "
370         "target-feature=+feature1,-feature2\n\n");
371}
372
373#else
374
375extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) {
376  printf("Target CPU help is not supported by this LLVM version.\n\n");
377}
378
379extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) {
380  printf("Target features help is not supported by this LLVM version.\n\n");
381}
382#endif
383
384extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
385  StringRef Name = sys::getHostCPUName();
386  *len = Name.size();
387  return Name.data();
388}
389
390extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
391    const char *TripleStr, const char *CPU, const char *Feature,
392    const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocMode RustReloc,
393    LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
394    bool PositionIndependentExecutable, bool FunctionSections,
395    bool DataSections,
396    bool TrapUnreachable,
397    bool Singlethread,
398    bool AsmComments,
399    bool EmitStackSizeSection,
400    bool RelaxELFRelocations) {
401
402  auto OptLevel = fromRust(RustOptLevel);
403  auto RM = fromRust(RustReloc);
404
405  std::string Error;
406  Triple Trip(Triple::normalize(TripleStr));
407  const llvm::Target *TheTarget =
408      TargetRegistry::lookupTarget(Trip.getTriple(), Error);
409  if (TheTarget == nullptr) {
410    LLVMRustSetLastError(Error.c_str());
411    return nullptr;
412  }
413
414  TargetOptions Options;
415
416  Options.FloatABIType = FloatABI::Default;
417  if (UseSoftFloat) {
418    Options.FloatABIType = FloatABI::Soft;
419  }
420  Options.DataSections = DataSections;
421  Options.FunctionSections = FunctionSections;
422  Options.MCOptions.AsmVerbose = AsmComments;
423  Options.MCOptions.PreserveAsmComments = AsmComments;
424  Options.MCOptions.ABIName = ABIStr;
425  Options.RelaxELFRelocations = RelaxELFRelocations;
426
427  if (TrapUnreachable) {
428    // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
429    // This limits the extent of possible undefined behavior in some cases, as
430    // it prevents control flow from "falling through" into whatever code
431    // happens to be laid out next in memory.
432    Options.TrapUnreachable = true;
433  }
434
435  if (Singlethread) {
436    Options.ThreadModel = ThreadModel::Single;
437  }
438
439  Options.EmitStackSizeSection = EmitStackSizeSection;
440
441  Optional<CodeModel::Model> CM;
442  if (RustCM != LLVMRustCodeModel::None)
443    CM = fromRust(RustCM);
444  TargetMachine *TM = TheTarget->createTargetMachine(
445      Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
446  return wrap(TM);
447}
448
449extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
450  delete unwrap(TM);
451}
452
453extern "C" void LLVMRustConfigurePassManagerBuilder(
454    LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
455    bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO,
456    const char* PGOGenPath, const char* PGOUsePath) {
457  unwrap(PMBR)->MergeFunctions = MergeFunctions;
458  unwrap(PMBR)->SLPVectorize = SLPVectorize;
459  unwrap(PMBR)->OptLevel = fromRust(OptLevel);
460  unwrap(PMBR)->LoopVectorize = LoopVectorize;
461  unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO;
462
463  if (PGOGenPath) {
464    assert(!PGOUsePath);
465    unwrap(PMBR)->EnablePGOInstrGen = true;
466    unwrap(PMBR)->PGOInstrGen = PGOGenPath;
467  }
468  if (PGOUsePath) {
469    assert(!PGOGenPath);
470    unwrap(PMBR)->PGOInstrUse = PGOUsePath;
471  }
472}
473
474// Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
475// field of a PassManagerBuilder, we expose our own method of doing so.
476extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
477                                              LLVMModuleRef M,
478                                              bool DisableSimplifyLibCalls) {
479  Triple TargetTriple(unwrap(M)->getTargetTriple());
480  TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple);
481  if (DisableSimplifyLibCalls)
482    TLI->disableAllFunctions();
483  unwrap(PMBR)->LibraryInfo = TLI;
484}
485
486// Unfortunately, the LLVM C API doesn't provide a way to create the
487// TargetLibraryInfo pass, so we use this method to do so.
488extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
489                                       bool DisableSimplifyLibCalls) {
490  Triple TargetTriple(unwrap(M)->getTargetTriple());
491  TargetLibraryInfoImpl TLII(TargetTriple);
492  if (DisableSimplifyLibCalls)
493    TLII.disableAllFunctions();
494  unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
495}
496
497// Unfortunately, the LLVM C API doesn't provide an easy way of iterating over
498// all the functions in a module, so we do that manually here. You'll find
499// similar code in clang's BackendUtil.cpp file.
500extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR,
501                                               LLVMModuleRef M) {
502  llvm::legacy::FunctionPassManager *P =
503      unwrap<llvm::legacy::FunctionPassManager>(PMR);
504  P->doInitialization();
505
506  // Upgrade all calls to old intrinsics first.
507  for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;)
508    UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
509
510  for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;
511       ++I)
512    if (!I->isDeclaration())
513      P->run(*I);
514
515  P->doFinalization();
516}
517
518extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
519  // Initializing the command-line options more than once is not allowed. So,
520  // check if they've already been initialized.  (This could happen if we're
521  // being called from rustpkg, for example). If the arguments change, then
522  // that's just kinda unfortunate.
523  static bool Initialized = false;
524  if (Initialized)
525    return;
526  Initialized = true;
527  cl::ParseCommandLineOptions(Argc, Argv);
528}
529
530enum class LLVMRustFileType {
531  Other,
532  AssemblyFile,
533  ObjectFile,
534};
535
536#if LLVM_VERSION_GE(10, 0)
537static CodeGenFileType fromRust(LLVMRustFileType Type) {
538  switch (Type) {
539  case LLVMRustFileType::AssemblyFile:
540    return CGFT_AssemblyFile;
541  case LLVMRustFileType::ObjectFile:
542    return CGFT_ObjectFile;
543  default:
544    report_fatal_error("Bad FileType.");
545  }
546}
547#else
548static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) {
549  switch (Type) {
550  case LLVMRustFileType::AssemblyFile:
551    return TargetMachine::CGFT_AssemblyFile;
552  case LLVMRustFileType::ObjectFile:
553    return TargetMachine::CGFT_ObjectFile;
554  default:
555    report_fatal_error("Bad FileType.");
556  }
557}
558#endif
559
560extern "C" LLVMRustResult
561LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
562                        LLVMModuleRef M, const char *Path,
563                        LLVMRustFileType RustFileType) {
564  llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
565  auto FileType = fromRust(RustFileType);
566
567  std::string ErrorInfo;
568  std::error_code EC;
569  raw_fd_ostream OS(Path, EC, sys::fs::F_None);
570  if (EC)
571    ErrorInfo = EC.message();
572  if (ErrorInfo != "") {
573    LLVMRustSetLastError(ErrorInfo.c_str());
574    return LLVMRustResult::Failure;
575  }
576
577  buffer_ostream BOS(OS);
578  unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
579  PM->run(*unwrap(M));
580
581  // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
582  // stream (OS), so the only real safe place to delete this is here? Don't we
583  // wish this was written in Rust?
584  LLVMDisposePassManager(PMR);
585  return LLVMRustResult::Success;
586}
587
588
589// Callback to demangle function name
590// Parameters:
591// * name to be demangled
592// * name len
593// * output buffer
594// * output buffer len
595// Returns len of demangled string, or 0 if demangle failed.
596typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
597
598
599namespace {
600
601class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
602  DemangleFn Demangle;
603  std::vector<char> Buf;
604
605public:
606  RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
607
608  // Return empty string if demangle failed
609  // or if name does not need to be demangled
610  StringRef CallDemangle(StringRef name) {
611    if (!Demangle) {
612      return StringRef();
613    }
614
615    if (Buf.size() < name.size() * 2) {
616      // Semangled name usually shorter than mangled,
617      // but allocate twice as much memory just in case
618      Buf.resize(name.size() * 2);
619    }
620
621    auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
622    if (!R) {
623      // Demangle failed.
624      return StringRef();
625    }
626
627    auto Demangled = StringRef(Buf.data(), R);
628    if (Demangled == name) {
629      // Do not print anything if demangled name is equal to mangled.
630      return StringRef();
631    }
632
633    return Demangled;
634  }
635
636  void emitFunctionAnnot(const Function *F,
637                         formatted_raw_ostream &OS) override {
638    StringRef Demangled = CallDemangle(F->getName());
639    if (Demangled.empty()) {
640        return;
641    }
642
643    OS << "; " << Demangled << "\n";
644  }
645
646  void emitInstructionAnnot(const Instruction *I,
647                            formatted_raw_ostream &OS) override {
648    const char *Name;
649    const Value *Value;
650    if (const CallInst *CI = dyn_cast<CallInst>(I)) {
651      Name = "call";
652      Value = CI->getCalledValue();
653    } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
654      Name = "invoke";
655      Value = II->getCalledValue();
656    } else {
657      // Could demangle more operations, e. g.
658      // `store %place, @function`.
659      return;
660    }
661
662    if (!Value->hasName()) {
663      return;
664    }
665
666    StringRef Demangled = CallDemangle(Value->getName());
667    if (Demangled.empty()) {
668      return;
669    }
670
671    OS << "; " << Name << " " << Demangled << "\n";
672  }
673};
674
675} // namespace
676
677extern "C" LLVMRustResult
678LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
679  std::string ErrorInfo;
680  std::error_code EC;
681  raw_fd_ostream OS(Path, EC, sys::fs::F_None);
682  if (EC)
683    ErrorInfo = EC.message();
684  if (ErrorInfo != "") {
685    LLVMRustSetLastError(ErrorInfo.c_str());
686    return LLVMRustResult::Failure;
687  }
688
689  RustAssemblyAnnotationWriter AAW(Demangle);
690  formatted_raw_ostream FOS(OS);
691  unwrap(M)->print(FOS, &AAW);
692
693  return LLVMRustResult::Success;
694}
695
696extern "C" void LLVMRustPrintPasses() {
697  LLVMInitializePasses();
698  struct MyListener : PassRegistrationListener {
699    void passEnumerate(const PassInfo *Info) {
700      StringRef PassArg = Info->getPassArgument();
701      StringRef PassName = Info->getPassName();
702      if (!PassArg.empty()) {
703        // These unsigned->signed casts could theoretically overflow, but
704        // realistically never will (and even if, the result is implementation
705        // defined rather plain UB).
706        printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
707               (int)PassName.size(), PassName.data());
708      }
709    }
710  } Listener;
711
712  PassRegistry *PR = PassRegistry::getPassRegistry();
713  PR->enumerateWith(&Listener);
714}
715
716extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR,
717                                            bool AddLifetimes) {
718  unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes);
719}
720
721extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
722                                           size_t Len) {
723  llvm::legacy::PassManager passes;
724
725  auto PreserveFunctions = [=](const GlobalValue &GV) {
726    for (size_t I = 0; I < Len; I++) {
727      if (GV.getName() == Symbols[I]) {
728        return true;
729      }
730    }
731    return false;
732  };
733
734  passes.add(llvm::createInternalizePass(PreserveFunctions));
735
736  passes.run(*unwrap(M));
737}
738
739extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) {
740  for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E;
741       ++GV) {
742    GV->setDoesNotThrow();
743    Function *F = dyn_cast<Function>(GV);
744    if (F == nullptr)
745      continue;
746
747    for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
748      for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) {
749        if (isa<InvokeInst>(I)) {
750          InvokeInst *CI = cast<InvokeInst>(I);
751          CI->setDoesNotThrow();
752        }
753      }
754    }
755  }
756}
757
758extern "C" void
759LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
760                                       LLVMTargetMachineRef TMR) {
761  TargetMachine *Target = unwrap(TMR);
762  unwrap(Module)->setDataLayout(Target->createDataLayout());
763}
764
765extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
766  unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
767}
768
769extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
770  unwrap(M)->setPIELevel(PIELevel::Level::Large);
771}
772
773// Here you'll find an implementation of ThinLTO as used by the Rust compiler
774// right now. This ThinLTO support is only enabled on "recent ish" versions of
775// LLVM, and otherwise it's just blanket rejected from other compilers.
776//
777// Most of this implementation is straight copied from LLVM. At the time of
778// this writing it wasn't *quite* suitable to reuse more code from upstream
779// for our purposes, but we should strive to upstream this support once it's
780// ready to go! I figure we may want a bit of testing locally first before
781// sending this upstream to LLVM. I hear though they're quite eager to receive
782// feedback like this!
783//
784// If you're reading this code and wondering "what in the world" or you're
785// working "good lord by LLVM upgrade is *still* failing due to these bindings"
786// then fear not! (ok maybe fear a little). All code here is mostly based
787// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
788//
789// You'll find that the general layout here roughly corresponds to the `run`
790// method in that file as well as `ProcessThinLTOModule`. Functions are
791// specifically commented below as well, but if you're updating this code
792// or otherwise trying to understand it, the LLVM source will be useful in
793// interpreting the mysteries within.
794//
795// Otherwise I'll apologize in advance, it probably requires a relatively
796// significant investment on your part to "truly understand" what's going on
797// here. Not saying I do myself, but it took me awhile staring at LLVM's source
798// and various online resources about ThinLTO to make heads or tails of all
799// this.
800
801// This is a shared data structure which *must* be threadsafe to share
802// read-only amongst threads. This also corresponds basically to the arguments
803// of the `ProcessThinLTOModule` function in the LLVM source.
804struct LLVMRustThinLTOData {
805  // The combined index that is the global analysis over all modules we're
806  // performing ThinLTO for. This is mostly managed by LLVM.
807  ModuleSummaryIndex Index;
808
809  // All modules we may look at, stored as in-memory serialized versions. This
810  // is later used when inlining to ensure we can extract any module to inline
811  // from.
812  StringMap<MemoryBufferRef> ModuleMap;
813
814  // A set that we manage of everything we *don't* want internalized. Note that
815  // this includes all transitive references right now as well, but it may not
816  // always!
817  DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
818
819  // Not 100% sure what these are, but they impact what's internalized and
820  // what's inlined across modules, I believe.
821  StringMap<FunctionImporter::ImportMapTy> ImportLists;
822  StringMap<FunctionImporter::ExportSetTy> ExportLists;
823  StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
824
825  LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
826};
827
828// Just an argument to the `LLVMRustCreateThinLTOData` function below.
829struct LLVMRustThinLTOModule {
830  const char *identifier;
831  const char *data;
832  size_t len;
833};
834
835// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
836// does.
837static const GlobalValueSummary *
838getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
839  auto StrongDefForLinker = llvm::find_if(
840      GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
841        auto Linkage = Summary->linkage();
842        return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
843               !GlobalValue::isWeakForLinker(Linkage);
844      });
845  if (StrongDefForLinker != GVSummaryList.end())
846    return StrongDefForLinker->get();
847
848  auto FirstDefForLinker = llvm::find_if(
849      GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
850        auto Linkage = Summary->linkage();
851        return !GlobalValue::isAvailableExternallyLinkage(Linkage);
852      });
853  if (FirstDefForLinker == GVSummaryList.end())
854    return nullptr;
855  return FirstDefForLinker->get();
856}
857
858// The main entry point for creating the global ThinLTO analysis. The structure
859// here is basically the same as before threads are spawned in the `run`
860// function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
861extern "C" LLVMRustThinLTOData*
862LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
863                          int num_modules,
864                          const char **preserved_symbols,
865                          int num_symbols) {
866#if LLVM_VERSION_GE(10, 0)
867  auto Ret = std::make_unique<LLVMRustThinLTOData>();
868#else
869  auto Ret = llvm::make_unique<LLVMRustThinLTOData>();
870#endif
871
872  // Load each module's summary and merge it into one combined index
873  for (int i = 0; i < num_modules; i++) {
874    auto module = &modules[i];
875    StringRef buffer(module->data, module->len);
876    MemoryBufferRef mem_buffer(buffer, module->identifier);
877
878    Ret->ModuleMap[module->identifier] = mem_buffer;
879
880    if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
881      LLVMRustSetLastError(toString(std::move(Err)).c_str());
882      return nullptr;
883    }
884  }
885
886  // Collect for each module the list of function it defines (GUID -> Summary)
887  Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
888
889  // Convert the preserved symbols set from string to GUID, this is then needed
890  // for internalization.
891  for (int i = 0; i < num_symbols; i++) {
892    auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
893    Ret->GUIDPreservedSymbols.insert(GUID);
894  }
895
896  // Collect the import/export lists for all modules from the call-graph in the
897  // combined index
898  //
899  // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
900  auto deadIsPrevailing = [&](GlobalValue::GUID G) {
901    return PrevailingType::Unknown;
902  };
903#if LLVM_VERSION_GE(8, 0)
904  // We don't have a complete picture in our use of ThinLTO, just our immediate
905  // crate, so we need `ImportEnabled = false` to limit internalization.
906  // Otherwise, we sometimes lose `static` values -- see #60184.
907  computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
908                                  deadIsPrevailing, /* ImportEnabled = */ false);
909#else
910  computeDeadSymbols(Ret->Index, Ret->GUIDPreservedSymbols, deadIsPrevailing);
911#endif
912  ComputeCrossModuleImport(
913    Ret->Index,
914    Ret->ModuleToDefinedGVSummaries,
915    Ret->ImportLists,
916    Ret->ExportLists
917  );
918
919  // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
920  // impacts the caching.
921  //
922  // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
923  // being lifted from `lib/LTO/LTO.cpp` as well
924  StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
925  DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
926  for (auto &I : Ret->Index) {
927    if (I.second.SummaryList.size() > 1)
928      PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
929  }
930  auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
931    const auto &Prevailing = PrevailingCopy.find(GUID);
932    if (Prevailing == PrevailingCopy.end())
933      return true;
934    return Prevailing->second == S;
935  };
936  auto recordNewLinkage = [&](StringRef ModuleIdentifier,
937                              GlobalValue::GUID GUID,
938                              GlobalValue::LinkageTypes NewLinkage) {
939    ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
940  };
941#if LLVM_VERSION_GE(9, 0)
942  thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage,
943                                  Ret->GUIDPreservedSymbols);
944#elif LLVM_VERSION_GE(8, 0)
945  thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage);
946#else
947  thinLTOResolveWeakForLinkerInIndex(Ret->Index, isPrevailing, recordNewLinkage);
948#endif
949
950  // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
951  // callback below. This callback below will dictate the linkage for all
952  // summaries in the index, and we basically just only want to ensure that dead
953  // symbols are internalized. Otherwise everything that's already external
954  // linkage will stay as external, and internal will stay as internal.
955  std::set<GlobalValue::GUID> ExportedGUIDs;
956  for (auto &List : Ret->Index) {
957    for (auto &GVS: List.second.SummaryList) {
958      if (GlobalValue::isLocalLinkage(GVS->linkage()))
959        continue;
960      auto GUID = GVS->getOriginalName();
961      if (GVS->flags().Live)
962        ExportedGUIDs.insert(GUID);
963    }
964  }
965#if LLVM_VERSION_GE(10, 0)
966  auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
967    const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
968    return (ExportList != Ret->ExportLists.end() &&
969      ExportList->second.count(VI)) ||
970      ExportedGUIDs.count(VI.getGUID());
971  };
972  thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
973#else
974  auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
975    const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
976    return (ExportList != Ret->ExportLists.end() &&
977      ExportList->second.count(GUID)) ||
978      ExportedGUIDs.count(GUID);
979  };
980  thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported);
981#endif
982
983  return Ret.release();
984}
985
986extern "C" void
987LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
988  delete Data;
989}
990
991// Below are the various passes that happen *per module* when doing ThinLTO.
992//
993// In other words, these are the functions that are all run concurrently
994// with one another, one per module. The passes here correspond to the analysis
995// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
996// `ProcessThinLTOModule` function. Here they're split up into separate steps
997// so rustc can save off the intermediate bytecode between each step.
998
999extern "C" bool
1000LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1001  Module &Mod = *unwrap(M);
1002  if (renameModuleForThinLTO(Mod, Data->Index)) {
1003    LLVMRustSetLastError("renameModuleForThinLTO failed");
1004    return false;
1005  }
1006  return true;
1007}
1008
1009extern "C" bool
1010LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1011  Module &Mod = *unwrap(M);
1012  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1013#if LLVM_VERSION_GE(8, 0)
1014  thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
1015#else
1016  thinLTOResolveWeakForLinkerModule(Mod, DefinedGlobals);
1017#endif
1018  return true;
1019}
1020
1021extern "C" bool
1022LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1023  Module &Mod = *unwrap(M);
1024  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1025  thinLTOInternalizeModule(Mod, DefinedGlobals);
1026  return true;
1027}
1028
1029extern "C" bool
1030LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1031  Module &Mod = *unwrap(M);
1032
1033  const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1034  auto Loader = [&](StringRef Identifier) {
1035    const auto &Memory = Data->ModuleMap.lookup(Identifier);
1036    auto &Context = Mod.getContext();
1037    auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1038
1039    if (!MOrErr)
1040      return MOrErr;
1041
1042    // The rest of this closure is a workaround for
1043    // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1044    // we accidentally import wasm custom sections into different modules,
1045    // duplicating them by in the final output artifact.
1046    //
1047    // The issue is worked around here by manually removing the
1048    // `wasm.custom_sections` named metadata node from any imported module. This
1049    // we know isn't used by any optimization pass so there's no need for it to
1050    // be imported.
1051    //
1052    // Note that the metadata is currently lazily loaded, so we materialize it
1053    // here before looking up if there's metadata inside. The `FunctionImporter`
1054    // will immediately materialize metadata anyway after an import, so this
1055    // shouldn't be a perf hit.
1056    if (Error Err = (*MOrErr)->materializeMetadata()) {
1057      Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1058      return Ret;
1059    }
1060
1061    auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1062    if (WasmCustomSections)
1063      WasmCustomSections->eraseFromParent();
1064
1065    return MOrErr;
1066  };
1067  FunctionImporter Importer(Data->Index, Loader);
1068  Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1069  if (!Result) {
1070    LLVMRustSetLastError(toString(Result.takeError()).c_str());
1071    return false;
1072  }
1073  return true;
1074}
1075
1076extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload
1077                                                      const char*, // importing module name
1078                                                      const char*); // imported module name
1079
1080// Calls `module_name_callback` for each module import done by ThinLTO.
1081// The callback is provided with regular null-terminated C strings.
1082extern "C" void
1083LLVMRustGetThinLTOModuleImports(const LLVMRustThinLTOData *data,
1084                                LLVMRustModuleNameCallback module_name_callback,
1085                                void* callback_payload) {
1086  for (const auto& importing_module : data->ImportLists) {
1087    const std::string importing_module_id = importing_module.getKey().str();
1088    const auto& imports = importing_module.getValue();
1089    for (const auto& imported_module : imports) {
1090      const std::string imported_module_id = imported_module.getKey().str();
1091      module_name_callback(callback_payload,
1092                           importing_module_id.c_str(),
1093                           imported_module_id.c_str());
1094    }
1095  }
1096}
1097
1098// This struct and various functions are sort of a hack right now, but the
1099// problem is that we've got in-memory LLVM modules after we generate and
1100// optimize all codegen-units for one compilation in rustc. To be compatible
1101// with the LTO support above we need to serialize the modules plus their
1102// ThinLTO summary into memory.
1103//
1104// This structure is basically an owned version of a serialize module, with
1105// a ThinLTO summary attached.
1106struct LLVMRustThinLTOBuffer {
1107  std::string data;
1108};
1109
1110extern "C" LLVMRustThinLTOBuffer*
1111LLVMRustThinLTOBufferCreate(LLVMModuleRef M) {
1112#if LLVM_VERSION_GE(10, 0)
1113  auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1114#else
1115  auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>();
1116#endif
1117  {
1118    raw_string_ostream OS(Ret->data);
1119    {
1120      legacy::PassManager PM;
1121      PM.add(createWriteThinLTOBitcodePass(OS));
1122      PM.run(*unwrap(M));
1123    }
1124  }
1125  return Ret.release();
1126}
1127
1128extern "C" void
1129LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1130  delete Buffer;
1131}
1132
1133extern "C" const void*
1134LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1135  return Buffer->data.data();
1136}
1137
1138extern "C" size_t
1139LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1140  return Buffer->data.length();
1141}
1142
1143// This is what we used to parse upstream bitcode for actual ThinLTO
1144// processing.  We'll call this once per module optimized through ThinLTO, and
1145// it'll be called concurrently on many threads.
1146extern "C" LLVMModuleRef
1147LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1148                           const char *data,
1149                           size_t len,
1150                           const char *identifier) {
1151  StringRef Data(data, len);
1152  MemoryBufferRef Buffer(Data, identifier);
1153  unwrap(Context)->enableDebugTypeODRUniquing();
1154  Expected<std::unique_ptr<Module>> SrcOrError =
1155      parseBitcodeFile(Buffer, *unwrap(Context));
1156  if (!SrcOrError) {
1157    LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1158    return nullptr;
1159  }
1160  return wrap(std::move(*SrcOrError).release());
1161}
1162
1163// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1164// the comment in `back/lto.rs` for why this exists.
1165extern "C" void
1166LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod,
1167                                DICompileUnit **A,
1168                                DICompileUnit **B) {
1169  Module *M = unwrap(Mod);
1170  DICompileUnit **Cur = A;
1171  DICompileUnit **Next = B;
1172  for (DICompileUnit *CU : M->debug_compile_units()) {
1173    *Cur = CU;
1174    Cur = Next;
1175    Next = nullptr;
1176    if (Cur == nullptr)
1177      break;
1178  }
1179}
1180
1181// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See
1182// the comment in `back/lto.rs` for why this exists.
1183extern "C" void
1184LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
1185  Module *M = unwrap(Mod);
1186
1187  // If the original source module didn't have a `DICompileUnit` then try to
1188  // merge all the existing compile units. If there aren't actually any though
1189  // then there's not much for us to do so return.
1190  if (Unit == nullptr) {
1191    for (DICompileUnit *CU : M->debug_compile_units()) {
1192      Unit = CU;
1193      break;
1194    }
1195    if (Unit == nullptr)
1196      return;
1197  }
1198
1199  // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
1200  // process it recursively. Note that we specifically iterate over instructions
1201  // to ensure we feed everything into it.
1202  DebugInfoFinder Finder;
1203  Finder.processModule(*M);
1204  for (Function &F : M->functions()) {
1205    for (auto &FI : F) {
1206      for (Instruction &BI : FI) {
1207        if (auto Loc = BI.getDebugLoc())
1208          Finder.processLocation(*M, Loc);
1209        if (auto DVI = dyn_cast<DbgValueInst>(&BI))
1210          Finder.processValue(*M, DVI);
1211        if (auto DDI = dyn_cast<DbgDeclareInst>(&BI))
1212          Finder.processDeclare(*M, DDI);
1213      }
1214    }
1215  }
1216
1217  // After we've found all our debuginfo, rewrite all subprograms to point to
1218  // the same `DICompileUnit`.
1219  for (auto &F : Finder.subprograms()) {
1220    F->replaceUnit(Unit);
1221  }
1222
1223  // Erase any other references to other `DICompileUnit` instances, the verifier
1224  // will later ensure that we don't actually have any other stale references to
1225  // worry about.
1226  auto *MD = M->getNamedMetadata("llvm.dbg.cu");
1227  MD->clearOperands();
1228  MD->addOperand(Unit);
1229}