fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #103046 - JanBeh:PR_clarify_cmp_terminology, r=workingjubilee3bcad65fbf6

master
55.7 KiB1654 linesraw
1#include <stdio.h>
2
3#include <cstddef>
4#include <iomanip>
5#include <vector>
6#include <set>
7
8#include "LLVMWrapper.h"
9
10#include "llvm/Analysis/AliasAnalysis.h"
11#include "llvm/Analysis/TargetLibraryInfo.h"
12#include "llvm/Analysis/TargetTransformInfo.h"
13#include "llvm/CodeGen/CommandFlags.h"
14#include "llvm/CodeGen/TargetSubtargetInfo.h"
15#include "llvm/IR/AutoUpgrade.h"
16#include "llvm/IR/AssemblyAnnotationWriter.h"
17#include "llvm/IR/IntrinsicInst.h"
18#include "llvm/IR/Verifier.h"
19#include "llvm/MC/TargetRegistry.h"
20#include "llvm/Object/ObjectFile.h"
21#include "llvm/Object/IRObjectFile.h"
22#include "llvm/Passes/PassBuilder.h"
23#include "llvm/Passes/PassPlugin.h"
24#include "llvm/Passes/StandardInstrumentations.h"
25#include "llvm/Support/CBindingWrapping.h"
26#include "llvm/Support/FileSystem.h"
27#if LLVM_VERSION_GE(17, 0)
28#include "llvm/Support/VirtualFileSystem.h"
29#endif
30#include "llvm/Target/TargetMachine.h"
31#include "llvm/Transforms/IPO/AlwaysInliner.h"
32#include "llvm/Transforms/IPO/FunctionImport.h"
33#include "llvm/Transforms/IPO/Internalize.h"
34#include "llvm/Transforms/IPO/LowerTypeTests.h"
35#include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
36#include "llvm/Transforms/Utils/AddDiscriminators.h"
37#include "llvm/Transforms/Utils/FunctionImportUtils.h"
38#include "llvm/LTO/LTO.h"
39#include "llvm/Bitcode/BitcodeWriter.h"
40
41#include "llvm/Transforms/Instrumentation.h"
42#include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
43#include "llvm/Support/TimeProfiler.h"
44#include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
45#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
46#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
47#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
48#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
49#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
50#include "llvm/Transforms/Utils/NameAnonGlobals.h"
51#include "llvm/Transforms/Utils.h"
52
53using namespace llvm;
54
55static codegen::RegisterCodeGenFlags CGF;
56
57typedef struct LLVMOpaquePass *LLVMPassRef;
58typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
59
60DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
61DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
62
63extern "C" void LLVMTimeTraceProfilerInitialize() {
64  timeTraceProfilerInitialize(
65      /* TimeTraceGranularity */ 0,
66      /* ProcName */ "rustc");
67}
68
69extern "C" void LLVMTimeTraceProfilerFinishThread() {
70  timeTraceProfilerFinishThread();
71}
72
73extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
74  StringRef FN(FileName);
75  std::error_code EC;
76  raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
77
78  timeTraceProfilerWrite(OS);
79  timeTraceProfilerCleanup();
80}
81
82#ifdef LLVM_COMPONENT_X86
83#define SUBTARGET_X86 SUBTARGET(X86)
84#else
85#define SUBTARGET_X86
86#endif
87
88#ifdef LLVM_COMPONENT_ARM
89#define SUBTARGET_ARM SUBTARGET(ARM)
90#else
91#define SUBTARGET_ARM
92#endif
93
94#ifdef LLVM_COMPONENT_AARCH64
95#define SUBTARGET_AARCH64 SUBTARGET(AArch64)
96#else
97#define SUBTARGET_AARCH64
98#endif
99
100#ifdef LLVM_COMPONENT_AVR
101#define SUBTARGET_AVR SUBTARGET(AVR)
102#else
103#define SUBTARGET_AVR
104#endif
105
106#ifdef LLVM_COMPONENT_M68k
107#define SUBTARGET_M68K SUBTARGET(M68k)
108#else
109#define SUBTARGET_M68K
110#endif
111
112#ifdef LLVM_COMPONENT_CSKY
113#define SUBTARGET_CSKY SUBTARGET(CSKY)
114#else
115#define SUBTARGET_CSKY
116#endif
117
118#ifdef LLVM_COMPONENT_MIPS
119#define SUBTARGET_MIPS SUBTARGET(Mips)
120#else
121#define SUBTARGET_MIPS
122#endif
123
124#ifdef LLVM_COMPONENT_POWERPC
125#define SUBTARGET_PPC SUBTARGET(PPC)
126#else
127#define SUBTARGET_PPC
128#endif
129
130#ifdef LLVM_COMPONENT_SYSTEMZ
131#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ)
132#else
133#define SUBTARGET_SYSTEMZ
134#endif
135
136#ifdef LLVM_COMPONENT_MSP430
137#define SUBTARGET_MSP430 SUBTARGET(MSP430)
138#else
139#define SUBTARGET_MSP430
140#endif
141
142#ifdef LLVM_COMPONENT_RISCV
143#define SUBTARGET_RISCV SUBTARGET(RISCV)
144#else
145#define SUBTARGET_RISCV
146#endif
147
148#ifdef LLVM_COMPONENT_SPARC
149#define SUBTARGET_SPARC SUBTARGET(Sparc)
150#else
151#define SUBTARGET_SPARC
152#endif
153
154#ifdef LLVM_COMPONENT_HEXAGON
155#define SUBTARGET_HEXAGON SUBTARGET(Hexagon)
156#else
157#define SUBTARGET_HEXAGON
158#endif
159
160#ifdef LLVM_COMPONENT_LOONGARCH
161#define SUBTARGET_LOONGARCH SUBTARGET(LoongArch)
162#else
163#define SUBTARGET_LOONGARCH
164#endif
165
166#define GEN_SUBTARGETS                                                         \
167  SUBTARGET_X86                                                                \
168  SUBTARGET_ARM                                                                \
169  SUBTARGET_AARCH64                                                            \
170  SUBTARGET_AVR                                                                \
171  SUBTARGET_M68K                                                               \
172  SUBTARGET_CSKY                                                               \
173  SUBTARGET_MIPS                                                               \
174  SUBTARGET_PPC                                                                \
175  SUBTARGET_SYSTEMZ                                                            \
176  SUBTARGET_MSP430                                                             \
177  SUBTARGET_SPARC                                                              \
178  SUBTARGET_HEXAGON                                                            \
179  SUBTARGET_RISCV                                                              \
180  SUBTARGET_LOONGARCH                                                          \
181
182#define SUBTARGET(x)                                                           \
183  namespace llvm {                                                             \
184  extern const SubtargetFeatureKV x##FeatureKV[];                              \
185  extern const SubtargetFeatureKV x##SubTypeKV[];                              \
186  }
187
188GEN_SUBTARGETS
189#undef SUBTARGET
190
191extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM,
192                                   const char *Feature) {
193  TargetMachine *Target = unwrap(TM);
194  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
195  return MCInfo->checkFeatures(std::string("+") + Feature);
196}
197
198enum class LLVMRustCodeModel {
199  Tiny,
200  Small,
201  Kernel,
202  Medium,
203  Large,
204  None,
205};
206
207#if LLVM_VERSION_LT(16, 0)
208static Optional<CodeModel::Model>
209#else
210static std::optional<CodeModel::Model>
211#endif
212fromRust(LLVMRustCodeModel Model) {
213  switch (Model) {
214  case LLVMRustCodeModel::Tiny:
215    return CodeModel::Tiny;
216  case LLVMRustCodeModel::Small:
217    return CodeModel::Small;
218  case LLVMRustCodeModel::Kernel:
219    return CodeModel::Kernel;
220  case LLVMRustCodeModel::Medium:
221    return CodeModel::Medium;
222  case LLVMRustCodeModel::Large:
223    return CodeModel::Large;
224  case LLVMRustCodeModel::None:
225#if LLVM_VERSION_LT(16, 0)
226    return None;
227#else
228    return std::nullopt;
229#endif
230  default:
231    report_fatal_error("Bad CodeModel.");
232  }
233}
234
235enum class LLVMRustCodeGenOptLevel {
236  None,
237  Less,
238  Default,
239  Aggressive,
240};
241
242#if LLVM_VERSION_GE(18, 0)
243  using CodeGenOptLevelEnum = llvm::CodeGenOptLevel;
244#else
245  using CodeGenOptLevelEnum = llvm::CodeGenOpt::Level;
246#endif
247
248static CodeGenOptLevelEnum fromRust(LLVMRustCodeGenOptLevel Level) {
249  switch (Level) {
250  case LLVMRustCodeGenOptLevel::None:
251    return CodeGenOptLevelEnum::None;
252  case LLVMRustCodeGenOptLevel::Less:
253    return CodeGenOptLevelEnum::Less;
254  case LLVMRustCodeGenOptLevel::Default:
255    return CodeGenOptLevelEnum::Default;
256  case LLVMRustCodeGenOptLevel::Aggressive:
257    return CodeGenOptLevelEnum::Aggressive;
258  default:
259    report_fatal_error("Bad CodeGenOptLevel.");
260  }
261}
262
263enum class LLVMRustPassBuilderOptLevel {
264  O0,
265  O1,
266  O2,
267  O3,
268  Os,
269  Oz,
270};
271
272static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
273  switch (Level) {
274  case LLVMRustPassBuilderOptLevel::O0:
275    return OptimizationLevel::O0;
276  case LLVMRustPassBuilderOptLevel::O1:
277    return OptimizationLevel::O1;
278  case LLVMRustPassBuilderOptLevel::O2:
279    return OptimizationLevel::O2;
280  case LLVMRustPassBuilderOptLevel::O3:
281    return OptimizationLevel::O3;
282  case LLVMRustPassBuilderOptLevel::Os:
283    return OptimizationLevel::Os;
284  case LLVMRustPassBuilderOptLevel::Oz:
285    return OptimizationLevel::Oz;
286  default:
287    report_fatal_error("Bad PassBuilderOptLevel.");
288  }
289}
290
291enum class LLVMRustRelocModel {
292  Static,
293  PIC,
294  DynamicNoPic,
295  ROPI,
296  RWPI,
297  ROPIRWPI,
298};
299
300static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) {
301  switch (RustReloc) {
302  case LLVMRustRelocModel::Static:
303    return Reloc::Static;
304  case LLVMRustRelocModel::PIC:
305    return Reloc::PIC_;
306  case LLVMRustRelocModel::DynamicNoPic:
307    return Reloc::DynamicNoPIC;
308  case LLVMRustRelocModel::ROPI:
309    return Reloc::ROPI;
310  case LLVMRustRelocModel::RWPI:
311    return Reloc::RWPI;
312  case LLVMRustRelocModel::ROPIRWPI:
313    return Reloc::ROPI_RWPI;
314  }
315  report_fatal_error("Bad RelocModel.");
316}
317
318/// getLongestEntryLength - Return the length of the longest entry in the table.
319template<typename KV>
320static size_t getLongestEntryLength(ArrayRef<KV> Table) {
321  size_t MaxLen = 0;
322  for (auto &I : Table)
323    MaxLen = std::max(MaxLen, std::strlen(I.Key));
324  return MaxLen;
325}
326
327using PrintBackendInfo = void(void*, const char* Data, size_t Len);
328
329extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM,
330                                        const char* TargetCPU,
331                                        PrintBackendInfo Print,
332                                        void* Out) {
333  const TargetMachine *Target = unwrap(TM);
334  const Triple::ArchType HostArch = Triple(sys::getDefaultTargetTriple()).getArch();
335  const Triple::ArchType TargetArch = Target->getTargetTriple().getArch();
336
337  std::ostringstream Buf;
338
339#if LLVM_VERSION_GE(17, 0)
340  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
341  const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getAllProcessorDescriptions();
342#else
343  Buf << "Full target CPU help is not supported by this LLVM version.\n\n";
344  SubtargetSubTypeKV TargetCPUKV = { TargetCPU, {{}}, {{}} };
345  const ArrayRef<SubtargetSubTypeKV> CPUTable = TargetCPUKV;
346#endif
347  unsigned MaxCPULen = getLongestEntryLength(CPUTable);
348
349  Buf << "Available CPUs for this target:\n";
350  // Don't print the "native" entry when the user specifies --target with a
351  // different arch since that could be wrong or misleading.
352  if (HostArch == TargetArch) {
353    MaxCPULen = std::max(MaxCPULen, (unsigned) std::strlen("native"));
354    const StringRef HostCPU = sys::getHostCPUName();
355    Buf << "    " << std::left << std::setw(MaxCPULen) << "native"
356        << " - Select the CPU of the current host "
357           "(currently " << HostCPU.str() << ").\n";
358  }
359  for (auto &CPU : CPUTable) {
360    // Compare cpu against current target to label the default
361    if (strcmp(CPU.Key, TargetCPU) == 0) {
362      Buf << "    " << std::left << std::setw(MaxCPULen) << CPU.Key
363          << " - This is the default target CPU for the current build target "
364             "(currently " << Target->getTargetTriple().str() << ").";
365    }
366    else {
367      Buf << "    " << CPU.Key;
368    }
369    Buf << "\n";
370  }
371
372  const auto &BufString = Buf.str();
373  Print(Out, BufString.data(), BufString.size());
374}
375
376extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) {
377#ifdef LLVM_RUSTLLVM
378  const TargetMachine *Target = unwrap(TM);
379  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
380  const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
381  return FeatTable.size();
382#else
383  return 0;
384#endif
385}
386
387extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index,
388                                         const char** Feature, const char** Desc) {
389#ifdef LLVM_RUSTLLVM
390  const TargetMachine *Target = unwrap(TM);
391  const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo();
392  const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable();
393  const SubtargetFeatureKV Feat = FeatTable[Index];
394  *Feature = Feat.Key;
395  *Desc = Feat.Desc;
396#endif
397}
398
399extern "C" const char* LLVMRustGetHostCPUName(size_t *len) {
400  StringRef Name = sys::getHostCPUName();
401  *len = Name.size();
402  return Name.data();
403}
404
405extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
406    const char *TripleStr, const char *CPU, const char *Feature,
407    const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc,
408    LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
409    bool FunctionSections,
410    bool DataSections,
411    bool UniqueSectionNames,
412    bool TrapUnreachable,
413    bool Singlethread,
414    bool AsmComments,
415    bool EmitStackSizeSection,
416    bool RelaxELFRelocations,
417    bool UseInitArray,
418    const char *SplitDwarfFile,
419    const char *OutputObjFile,
420    const char *DebugInfoCompression,
421    bool ForceEmulatedTls,
422    const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) {
423
424  auto OptLevel = fromRust(RustOptLevel);
425  auto RM = fromRust(RustReloc);
426  auto CM = fromRust(RustCM);
427
428  std::string Error;
429  Triple Trip(Triple::normalize(TripleStr));
430  const llvm::Target *TheTarget =
431      TargetRegistry::lookupTarget(Trip.getTriple(), Error);
432  if (TheTarget == nullptr) {
433    LLVMRustSetLastError(Error.c_str());
434    return nullptr;
435  }
436
437  TargetOptions Options = codegen::InitTargetOptionsFromCodeGenFlags(Trip);
438
439  Options.FloatABIType = FloatABI::Default;
440  if (UseSoftFloat) {
441    Options.FloatABIType = FloatABI::Soft;
442  }
443  Options.DataSections = DataSections;
444  Options.FunctionSections = FunctionSections;
445  Options.UniqueSectionNames = UniqueSectionNames;
446  Options.MCOptions.AsmVerbose = AsmComments;
447  Options.MCOptions.PreserveAsmComments = AsmComments;
448  Options.MCOptions.ABIName = ABIStr;
449  if (SplitDwarfFile) {
450      Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
451  }
452  if (OutputObjFile) {
453      Options.ObjectFilenameForDebug = OutputObjFile;
454  }
455#if LLVM_VERSION_GE(16, 0)
456  if (!strcmp("zlib", DebugInfoCompression) && llvm::compression::zlib::isAvailable()) {
457    Options.CompressDebugSections = DebugCompressionType::Zlib;
458  } else if (!strcmp("zstd", DebugInfoCompression) && llvm::compression::zstd::isAvailable()) {
459    Options.CompressDebugSections = DebugCompressionType::Zstd;
460  } else if (!strcmp("none", DebugInfoCompression)) {
461    Options.CompressDebugSections = DebugCompressionType::None;
462  }
463#endif
464
465  Options.RelaxELFRelocations = RelaxELFRelocations;
466  Options.UseInitArray = UseInitArray;
467
468#if LLVM_VERSION_LT(17, 0)
469  if (ForceEmulatedTls) {
470    Options.ExplicitEmulatedTLS = true;
471    Options.EmulatedTLS = true;
472  }
473#else
474  Options.EmulatedTLS = ForceEmulatedTls || Trip.hasDefaultEmulatedTLS();
475#endif
476
477  if (TrapUnreachable) {
478    // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
479    // This limits the extent of possible undefined behavior in some cases, as
480    // it prevents control flow from "falling through" into whatever code
481    // happens to be laid out next in memory.
482    Options.TrapUnreachable = true;
483  }
484
485  if (Singlethread) {
486    Options.ThreadModel = ThreadModel::Single;
487  }
488
489  Options.EmitStackSizeSection = EmitStackSizeSection;
490
491
492  if (ArgsCstrBuff != nullptr)
493  {
494    int buffer_offset = 0;
495    assert(ArgsCstrBuff[ArgsCstrBuffLen - 1] == '\0');
496
497    const size_t arg0_len = std::strlen(ArgsCstrBuff);
498    char* arg0 = new char[arg0_len + 1];
499    memcpy(arg0, ArgsCstrBuff, arg0_len);
500    arg0[arg0_len] = '\0';
501    buffer_offset += arg0_len + 1;
502
503    const int num_cmd_arg_strings =
504      std::count(&ArgsCstrBuff[buffer_offset], &ArgsCstrBuff[ArgsCstrBuffLen], '\0');
505
506    std::string* cmd_arg_strings = new std::string[num_cmd_arg_strings];
507    for (int i = 0; i < num_cmd_arg_strings; ++i)
508    {
509      assert(buffer_offset < ArgsCstrBuffLen);
510      const int len = std::strlen(ArgsCstrBuff + buffer_offset);
511      cmd_arg_strings[i] = std::string(&ArgsCstrBuff[buffer_offset], len);
512      buffer_offset += len + 1;
513    }
514
515    assert(buffer_offset == ArgsCstrBuffLen);
516
517    Options.MCOptions.Argv0 = arg0;
518    Options.MCOptions.CommandLineArgs =
519      llvm::ArrayRef<std::string>(cmd_arg_strings, num_cmd_arg_strings);
520  }
521
522  TargetMachine *TM = TheTarget->createTargetMachine(
523      Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel);
524  return wrap(TM);
525}
526
527extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) {
528
529  MCTargetOptions& MCOptions = unwrap(TM)->Options.MCOptions;
530  delete[] MCOptions.Argv0;
531  delete[] MCOptions.CommandLineArgs.data();
532
533  delete unwrap(TM);
534}
535
536// Unfortunately, the LLVM C API doesn't provide a way to create the
537// TargetLibraryInfo pass, so we use this method to do so.
538extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M,
539                                       bool DisableSimplifyLibCalls) {
540  Triple TargetTriple(unwrap(M)->getTargetTriple());
541  TargetLibraryInfoImpl TLII(TargetTriple);
542  if (DisableSimplifyLibCalls)
543    TLII.disableAllFunctions();
544  unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII));
545}
546
547extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) {
548  // Initializing the command-line options more than once is not allowed. So,
549  // check if they've already been initialized. (This could happen if we're
550  // being called from rustpkg, for example). If the arguments change, then
551  // that's just kinda unfortunate.
552  static bool Initialized = false;
553  if (Initialized)
554    return;
555  Initialized = true;
556  cl::ParseCommandLineOptions(Argc, Argv);
557}
558
559enum class LLVMRustFileType {
560  AssemblyFile,
561  ObjectFile,
562};
563
564static CodeGenFileType fromRust(LLVMRustFileType Type) {
565  switch (Type) {
566  case LLVMRustFileType::AssemblyFile:
567#if LLVM_VERSION_GE(18, 0)
568    return CodeGenFileType::AssemblyFile;
569#else
570    return CGFT_AssemblyFile;
571#endif
572  case LLVMRustFileType::ObjectFile:
573#if LLVM_VERSION_GE(18, 0)
574    return CodeGenFileType::ObjectFile;
575#else
576    return CGFT_ObjectFile;
577#endif
578  default:
579    report_fatal_error("Bad FileType.");
580  }
581}
582
583extern "C" LLVMRustResult
584LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
585                        LLVMModuleRef M, const char *Path, const char *DwoPath,
586                        LLVMRustFileType RustFileType) {
587  llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR);
588  auto FileType = fromRust(RustFileType);
589
590  std::string ErrorInfo;
591  std::error_code EC;
592  raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
593  if (EC)
594    ErrorInfo = EC.message();
595  if (ErrorInfo != "") {
596    LLVMRustSetLastError(ErrorInfo.c_str());
597    return LLVMRustResult::Failure;
598  }
599
600  buffer_ostream BOS(OS);
601  if (DwoPath) {
602    raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None);
603    EC.clear();
604    if (EC)
605        ErrorInfo = EC.message();
606    if (ErrorInfo != "") {
607      LLVMRustSetLastError(ErrorInfo.c_str());
608      return LLVMRustResult::Failure;
609    }
610    buffer_ostream DBOS(DOS);
611    unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false);
612    PM->run(*unwrap(M));
613  } else {
614    unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false);
615    PM->run(*unwrap(M));
616  }
617
618  // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output
619  // stream (OS), so the only real safe place to delete this is here? Don't we
620  // wish this was written in Rust?
621  LLVMDisposePassManager(PMR);
622  return LLVMRustResult::Success;
623}
624
625extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
626                                                      const char*,      // pass name
627                                                      const char*);     // IR name
628extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
629
630std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
631  if (const auto *Cast = any_cast<const Module *>(&WrappedIr))
632    return (*Cast)->getName().str();
633  if (const auto *Cast = any_cast<const Function *>(&WrappedIr))
634    return (*Cast)->getName().str();
635  if (const auto *Cast = any_cast<const Loop *>(&WrappedIr))
636    return (*Cast)->getName().str();
637  if (const auto *Cast = any_cast<const LazyCallGraph::SCC *>(&WrappedIr))
638    return (*Cast)->getName();
639  return "<UNKNOWN>";
640}
641
642
643void LLVMSelfProfileInitializeCallbacks(
644    PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
645    LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
646    LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
647  PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback](
648                                           StringRef Pass, llvm::Any Ir) {
649    std::string PassName = Pass.str();
650    std::string IrName = LLVMRustwrappedIrGetName(Ir);
651    BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
652  });
653
654  PIC.registerAfterPassCallback(
655      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR,
656                                            const PreservedAnalyses &Preserved) {
657        AfterPassCallback(LlvmSelfProfiler);
658      });
659
660  PIC.registerAfterPassInvalidatedCallback(
661      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) {
662        AfterPassCallback(LlvmSelfProfiler);
663      });
664
665  PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
666                                         StringRef Pass, llvm::Any Ir) {
667    std::string PassName = Pass.str();
668    std::string IrName = LLVMRustwrappedIrGetName(Ir);
669    BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
670  });
671
672  PIC.registerAfterAnalysisCallback(
673      [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
674        AfterPassCallback(LlvmSelfProfiler);
675      });
676}
677
678enum class LLVMRustOptStage {
679  PreLinkNoLTO,
680  PreLinkThinLTO,
681  PreLinkFatLTO,
682  ThinLTO,
683  FatLTO,
684};
685
686struct LLVMRustSanitizerOptions {
687  bool SanitizeAddress;
688  bool SanitizeAddressRecover;
689  bool SanitizeCFI;
690  bool SanitizeKCFI;
691  bool SanitizeMemory;
692  bool SanitizeMemoryRecover;
693  int  SanitizeMemoryTrackOrigins;
694  bool SanitizeThread;
695  bool SanitizeHWAddress;
696  bool SanitizeHWAddressRecover;
697  bool SanitizeKernelAddress;
698  bool SanitizeKernelAddressRecover;
699};
700
701extern "C" LLVMRustResult
702LLVMRustOptimize(
703    LLVMModuleRef ModuleRef,
704    LLVMTargetMachineRef TMRef,
705    LLVMRustPassBuilderOptLevel OptLevelRust,
706    LLVMRustOptStage OptStage,
707    bool IsLinkerPluginLTO,
708    bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
709    bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
710    bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers,
711    LLVMRustSanitizerOptions *SanitizerOptions,
712    const char *PGOGenPath, const char *PGOUsePath,
713    bool InstrumentCoverage, const char *InstrProfileOutput,
714    bool InstrumentGCOV,
715    const char *PGOSampleUsePath, bool DebugInfoForProfiling,
716    void* LlvmSelfProfiler,
717    LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
718    LLVMRustSelfProfileAfterPassCallback AfterPassCallback,
719    const char *ExtraPasses, size_t ExtraPassesLen,
720    const char *LLVMPlugins, size_t LLVMPluginsLen) {
721  Module *TheModule = unwrap(ModuleRef);
722  TargetMachine *TM = unwrap(TMRef);
723  OptimizationLevel OptLevel = fromRust(OptLevelRust);
724
725
726  PipelineTuningOptions PTO;
727  PTO.LoopUnrolling = UnrollLoops;
728  PTO.LoopInterleaving = UnrollLoops;
729  PTO.LoopVectorization = LoopVectorize;
730  PTO.SLPVectorization = SLPVectorize;
731  PTO.MergeFunctions = MergeFunctions;
732
733  // FIXME: We may want to expose this as an option.
734  bool DebugPassManager = false;
735
736  PassInstrumentationCallbacks PIC;
737#if LLVM_VERSION_LT(16, 0)
738  StandardInstrumentations SI(DebugPassManager);
739#else
740  StandardInstrumentations SI(TheModule->getContext(), DebugPassManager);
741#endif
742  SI.registerCallbacks(PIC);
743
744  if (LlvmSelfProfiler){
745    LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
746  }
747
748#if LLVM_VERSION_LT(16, 0)
749  Optional<PGOOptions> PGOOpt;
750#else
751  std::optional<PGOOptions> PGOOpt;
752#endif
753#if LLVM_VERSION_GE(17, 0)
754  auto FS = vfs::getRealFileSystem();
755#endif
756  if (PGOGenPath) {
757    assert(!PGOUsePath && !PGOSampleUsePath);
758    PGOOpt = PGOOptions(PGOGenPath, "", "",
759#if LLVM_VERSION_GE(17, 0)
760                        "",
761                        FS,
762#endif
763                        PGOOptions::IRInstr, PGOOptions::NoCSAction,
764                        DebugInfoForProfiling);
765  } else if (PGOUsePath) {
766    assert(!PGOSampleUsePath);
767    PGOOpt = PGOOptions(PGOUsePath, "", "",
768#if LLVM_VERSION_GE(17, 0)
769                        "",
770                        FS,
771#endif
772                        PGOOptions::IRUse, PGOOptions::NoCSAction,
773                        DebugInfoForProfiling);
774  } else if (PGOSampleUsePath) {
775    PGOOpt = PGOOptions(PGOSampleUsePath, "", "",
776#if LLVM_VERSION_GE(17, 0)
777                        "",
778                        FS,
779#endif
780                        PGOOptions::SampleUse, PGOOptions::NoCSAction,
781                        DebugInfoForProfiling);
782  } else if (DebugInfoForProfiling) {
783    PGOOpt = PGOOptions("", "", "",
784#if LLVM_VERSION_GE(17, 0)
785                        "",
786                        FS,
787#endif
788                        PGOOptions::NoAction, PGOOptions::NoCSAction,
789                        DebugInfoForProfiling);
790  }
791
792  PassBuilder PB(TM, PTO, PGOOpt, &PIC);
793  LoopAnalysisManager LAM;
794  FunctionAnalysisManager FAM;
795  CGSCCAnalysisManager CGAM;
796  ModuleAnalysisManager MAM;
797
798  FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
799
800  Triple TargetTriple(TheModule->getTargetTriple());
801  std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
802  if (DisableSimplifyLibCalls)
803    TLII->disableAllFunctions();
804  FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
805
806  PB.registerModuleAnalyses(MAM);
807  PB.registerCGSCCAnalyses(CGAM);
808  PB.registerFunctionAnalyses(FAM);
809  PB.registerLoopAnalyses(LAM);
810  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
811
812  // We manually collect pipeline callbacks so we can apply them at O0, where the
813  // PassBuilder does not create a pipeline.
814  std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
815      PipelineStartEPCallbacks;
816  std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>>
817      OptimizerLastEPCallbacks;
818
819  if (!IsLinkerPluginLTO
820      && SanitizerOptions && SanitizerOptions->SanitizeCFI
821      && !NoPrepopulatePasses) {
822    PipelineStartEPCallbacks.push_back(
823      [](ModulePassManager &MPM, OptimizationLevel Level) {
824        MPM.addPass(LowerTypeTestsPass(/*ExportSummary=*/nullptr,
825                                       /*ImportSummary=*/nullptr,
826                                       /*DropTypeTests=*/false));
827      }
828    );
829  }
830
831  if (VerifyIR) {
832    PipelineStartEPCallbacks.push_back(
833      [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) {
834        MPM.addPass(VerifierPass());
835      }
836    );
837  }
838
839  if (InstrumentGCOV) {
840    PipelineStartEPCallbacks.push_back(
841      [](ModulePassManager &MPM, OptimizationLevel Level) {
842        MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault()));
843      }
844    );
845  }
846
847  if (InstrumentCoverage) {
848    PipelineStartEPCallbacks.push_back(
849      [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) {
850        InstrProfOptions Options;
851        if (InstrProfileOutput) {
852          Options.InstrProfileOutput = InstrProfileOutput;
853        }
854        // cargo run tests in multhreading mode by default
855        // so use atomics for coverage counters
856        Options.Atomic = true;
857        MPM.addPass(InstrProfiling(Options, false));
858      }
859    );
860  }
861
862  if (SanitizerOptions) {
863    if (SanitizerOptions->SanitizeMemory) {
864      MemorySanitizerOptions Options(
865          SanitizerOptions->SanitizeMemoryTrackOrigins,
866          SanitizerOptions->SanitizeMemoryRecover,
867          /*CompileKernel=*/false,
868          /*EagerChecks=*/true);
869      OptimizerLastEPCallbacks.push_back(
870        [Options](ModulePassManager &MPM, OptimizationLevel Level) {
871#if LLVM_VERSION_LT(16, 0)
872          MPM.addPass(ModuleMemorySanitizerPass(Options));
873          MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
874#else
875          MPM.addPass(MemorySanitizerPass(Options));
876#endif
877        }
878      );
879    }
880
881    if (SanitizerOptions->SanitizeThread) {
882      OptimizerLastEPCallbacks.push_back(
883        [](ModulePassManager &MPM, OptimizationLevel Level) {
884          MPM.addPass(ModuleThreadSanitizerPass());
885          MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
886        }
887      );
888    }
889
890    if (SanitizerOptions->SanitizeAddress || SanitizerOptions->SanitizeKernelAddress) {
891      OptimizerLastEPCallbacks.push_back(
892        [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
893          auto CompileKernel = SanitizerOptions->SanitizeKernelAddress;
894          AddressSanitizerOptions opts = AddressSanitizerOptions{
895            CompileKernel,
896            SanitizerOptions->SanitizeAddressRecover
897              || SanitizerOptions->SanitizeKernelAddressRecover,
898            /*UseAfterScope=*/true,
899            AsanDetectStackUseAfterReturnMode::Runtime,
900          };
901#if LLVM_VERSION_LT(16, 0)
902          MPM.addPass(ModuleAddressSanitizerPass(opts));
903#else
904          MPM.addPass(AddressSanitizerPass(opts));
905#endif
906        }
907      );
908    }
909    if (SanitizerOptions->SanitizeHWAddress) {
910      OptimizerLastEPCallbacks.push_back(
911        [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
912          HWAddressSanitizerOptions opts(
913              /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover,
914              /*DisableOptimization=*/false);
915          MPM.addPass(HWAddressSanitizerPass(opts));
916        }
917      );
918    }
919  }
920
921  if (LLVMPluginsLen) {
922    auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen);
923    SmallVector<StringRef> Plugins;
924    PluginsStr.split(Plugins, ',', -1, false);
925    for (auto PluginPath: Plugins) {
926      auto Plugin = PassPlugin::Load(PluginPath.str());
927      if (!Plugin) {
928        LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str());
929        return LLVMRustResult::Failure;
930      }
931      Plugin->registerPassBuilderCallbacks(PB);
932    }
933  }
934
935  ModulePassManager MPM;
936  bool NeedThinLTOBufferPasses = UseThinLTOBuffers;
937  if (!NoPrepopulatePasses) {
938    // The pre-link pipelines don't support O0 and require using buildO0DefaultPipeline() instead.
939    // At the same time, the LTO pipelines do support O0 and using them is required.
940    bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO;
941    if (OptLevel == OptimizationLevel::O0 && !IsLTO) {
942      for (const auto &C : PipelineStartEPCallbacks)
943        PB.registerPipelineStartEPCallback(C);
944      for (const auto &C : OptimizerLastEPCallbacks)
945        PB.registerOptimizerLastEPCallback(C);
946
947      // Pass false as we manually schedule ThinLTOBufferPasses below.
948      MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false);
949    } else {
950      for (const auto &C : PipelineStartEPCallbacks)
951        PB.registerPipelineStartEPCallback(C);
952      if (OptStage != LLVMRustOptStage::PreLinkThinLTO) {
953        for (const auto &C : OptimizerLastEPCallbacks)
954          PB.registerOptimizerLastEPCallback(C);
955      }
956
957      switch (OptStage) {
958      case LLVMRustOptStage::PreLinkNoLTO:
959        MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
960        break;
961      case LLVMRustOptStage::PreLinkThinLTO:
962        MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel);
963        // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback
964        // passes may still run afterwards. This means we need to run the buffer passes again.
965        // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks
966        // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks.
967        if (OptimizerLastEPCallbacks.empty())
968          NeedThinLTOBufferPasses = false;
969        for (const auto &C : OptimizerLastEPCallbacks)
970          C(MPM, OptLevel);
971        break;
972      case LLVMRustOptStage::PreLinkFatLTO:
973        MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel);
974        NeedThinLTOBufferPasses = false;
975        break;
976      case LLVMRustOptStage::ThinLTO:
977        // FIXME: Does it make sense to pass the ModuleSummaryIndex?
978        // It only seems to be needed for C++ specific optimizations.
979        MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr);
980        break;
981      case LLVMRustOptStage::FatLTO:
982        MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr);
983        break;
984      }
985    }
986  } else {
987    // We're not building any of the default pipelines but we still want to
988    // add the verifier, instrumentation, etc passes if they were requested
989    for (const auto &C : PipelineStartEPCallbacks)
990      C(MPM, OptLevel);
991    for (const auto &C : OptimizerLastEPCallbacks)
992      C(MPM, OptLevel);
993  }
994
995  if (ExtraPassesLen) {
996    if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) {
997      std::string ErrMsg = toString(std::move(Err));
998      LLVMRustSetLastError(ErrMsg.c_str());
999      return LLVMRustResult::Failure;
1000    }
1001  }
1002
1003  if (NeedThinLTOBufferPasses) {
1004    MPM.addPass(CanonicalizeAliasesPass());
1005    MPM.addPass(NameAnonGlobalPass());
1006  }
1007
1008  // Upgrade all calls to old intrinsics first.
1009  for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
1010    UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
1011
1012  MPM.run(*TheModule, MAM);
1013  return LLVMRustResult::Success;
1014}
1015
1016// Callback to demangle function name
1017// Parameters:
1018// * name to be demangled
1019// * name len
1020// * output buffer
1021// * output buffer len
1022// Returns len of demangled string, or 0 if demangle failed.
1023typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t);
1024
1025
1026namespace {
1027
1028class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
1029  DemangleFn Demangle;
1030  std::vector<char> Buf;
1031
1032public:
1033  RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {}
1034
1035  // Return empty string if demangle failed
1036  // or if name does not need to be demangled
1037  StringRef CallDemangle(StringRef name) {
1038    if (!Demangle) {
1039      return StringRef();
1040    }
1041
1042    if (Buf.size() < name.size() * 2) {
1043      // Semangled name usually shorter than mangled,
1044      // but allocate twice as much memory just in case
1045      Buf.resize(name.size() * 2);
1046    }
1047
1048    auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size());
1049    if (!R) {
1050      // Demangle failed.
1051      return StringRef();
1052    }
1053
1054    auto Demangled = StringRef(Buf.data(), R);
1055    if (Demangled == name) {
1056      // Do not print anything if demangled name is equal to mangled.
1057      return StringRef();
1058    }
1059
1060    return Demangled;
1061  }
1062
1063  void emitFunctionAnnot(const Function *F,
1064                         formatted_raw_ostream &OS) override {
1065    StringRef Demangled = CallDemangle(F->getName());
1066    if (Demangled.empty()) {
1067        return;
1068    }
1069
1070    OS << "; " << Demangled << "\n";
1071  }
1072
1073  void emitInstructionAnnot(const Instruction *I,
1074                            formatted_raw_ostream &OS) override {
1075    const char *Name;
1076    const Value *Value;
1077    if (const CallInst *CI = dyn_cast<CallInst>(I)) {
1078      Name = "call";
1079      Value = CI->getCalledOperand();
1080    } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
1081      Name = "invoke";
1082      Value = II->getCalledOperand();
1083    } else {
1084      // Could demangle more operations, e. g.
1085      // `store %place, @function`.
1086      return;
1087    }
1088
1089    if (!Value->hasName()) {
1090      return;
1091    }
1092
1093    StringRef Demangled = CallDemangle(Value->getName());
1094    if (Demangled.empty()) {
1095      return;
1096    }
1097
1098    OS << "; " << Name << " " << Demangled << "\n";
1099  }
1100};
1101
1102} // namespace
1103
1104extern "C" LLVMRustResult
1105LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
1106  std::string ErrorInfo;
1107  std::error_code EC;
1108  raw_fd_ostream OS(Path, EC, sys::fs::OF_None);
1109  if (EC)
1110    ErrorInfo = EC.message();
1111  if (ErrorInfo != "") {
1112    LLVMRustSetLastError(ErrorInfo.c_str());
1113    return LLVMRustResult::Failure;
1114  }
1115
1116  RustAssemblyAnnotationWriter AAW(Demangle);
1117  formatted_raw_ostream FOS(OS);
1118  unwrap(M)->print(FOS, &AAW);
1119
1120  return LLVMRustResult::Success;
1121}
1122
1123extern "C" void LLVMRustPrintPasses() {
1124  PassBuilder PB;
1125  PB.printPassNames(outs());
1126}
1127
1128extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
1129                                           size_t Len) {
1130  auto PreserveFunctions = [=](const GlobalValue &GV) {
1131    // Preserve LLVM-injected, ASAN-related symbols.
1132    // See also https://github.com/rust-lang/rust/issues/113404.
1133    if (GV.getName() == "___asan_globals_registered") {
1134      return true;
1135    }
1136
1137    // Preserve symbols exported from Rust modules.
1138    for (size_t I = 0; I < Len; I++) {
1139      if (GV.getName() == Symbols[I]) {
1140        return true;
1141      }
1142    }
1143    return false;
1144  };
1145
1146  internalizeModule(*unwrap(M), PreserveFunctions);
1147}
1148
1149extern "C" void
1150LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module,
1151                                       LLVMTargetMachineRef TMR) {
1152  TargetMachine *Target = unwrap(TMR);
1153  unwrap(Module)->setDataLayout(Target->createDataLayout());
1154}
1155
1156extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) {
1157  unwrap(M)->setPICLevel(PICLevel::Level::BigPIC);
1158}
1159
1160extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) {
1161  unwrap(M)->setPIELevel(PIELevel::Level::Large);
1162}
1163
1164extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M,
1165                                           LLVMRustCodeModel Model) {
1166  auto CM = fromRust(Model);
1167  if (!CM)
1168    return;
1169  unwrap(M)->setCodeModel(*CM);
1170}
1171
1172// Here you'll find an implementation of ThinLTO as used by the Rust compiler
1173// right now. This ThinLTO support is only enabled on "recent ish" versions of
1174// LLVM, and otherwise it's just blanket rejected from other compilers.
1175//
1176// Most of this implementation is straight copied from LLVM. At the time of
1177// this writing it wasn't *quite* suitable to reuse more code from upstream
1178// for our purposes, but we should strive to upstream this support once it's
1179// ready to go! I figure we may want a bit of testing locally first before
1180// sending this upstream to LLVM. I hear though they're quite eager to receive
1181// feedback like this!
1182//
1183// If you're reading this code and wondering "what in the world" or you're
1184// working "good lord by LLVM upgrade is *still* failing due to these bindings"
1185// then fear not! (ok maybe fear a little). All code here is mostly based
1186// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM.
1187//
1188// You'll find that the general layout here roughly corresponds to the `run`
1189// method in that file as well as `ProcessThinLTOModule`. Functions are
1190// specifically commented below as well, but if you're updating this code
1191// or otherwise trying to understand it, the LLVM source will be useful in
1192// interpreting the mysteries within.
1193//
1194// Otherwise I'll apologize in advance, it probably requires a relatively
1195// significant investment on your part to "truly understand" what's going on
1196// here. Not saying I do myself, but it took me awhile staring at LLVM's source
1197// and various online resources about ThinLTO to make heads or tails of all
1198// this.
1199
1200// This is a shared data structure which *must* be threadsafe to share
1201// read-only amongst threads. This also corresponds basically to the arguments
1202// of the `ProcessThinLTOModule` function in the LLVM source.
1203struct LLVMRustThinLTOData {
1204  // The combined index that is the global analysis over all modules we're
1205  // performing ThinLTO for. This is mostly managed by LLVM.
1206  ModuleSummaryIndex Index;
1207
1208  // All modules we may look at, stored as in-memory serialized versions. This
1209  // is later used when inlining to ensure we can extract any module to inline
1210  // from.
1211  StringMap<MemoryBufferRef> ModuleMap;
1212
1213  // A set that we manage of everything we *don't* want internalized. Note that
1214  // this includes all transitive references right now as well, but it may not
1215  // always!
1216  DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1217
1218  // Not 100% sure what these are, but they impact what's internalized and
1219  // what's inlined across modules, I believe.
1220#if LLVM_VERSION_GE(18, 0)
1221  DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists;
1222  DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists;
1223  DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries;
1224#else
1225  StringMap<FunctionImporter::ImportMapTy> ImportLists;
1226  StringMap<FunctionImporter::ExportSetTy> ExportLists;
1227  StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
1228#endif
1229  StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1230
1231  LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {}
1232};
1233
1234// Just an argument to the `LLVMRustCreateThinLTOData` function below.
1235struct LLVMRustThinLTOModule {
1236  const char *identifier;
1237  const char *data;
1238  size_t len;
1239};
1240
1241// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it
1242// does.
1243static const GlobalValueSummary *
1244getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
1245  auto StrongDefForLinker = llvm::find_if(
1246      GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1247        auto Linkage = Summary->linkage();
1248        return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
1249               !GlobalValue::isWeakForLinker(Linkage);
1250      });
1251  if (StrongDefForLinker != GVSummaryList.end())
1252    return StrongDefForLinker->get();
1253
1254  auto FirstDefForLinker = llvm::find_if(
1255      GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
1256        auto Linkage = Summary->linkage();
1257        return !GlobalValue::isAvailableExternallyLinkage(Linkage);
1258      });
1259  if (FirstDefForLinker == GVSummaryList.end())
1260    return nullptr;
1261  return FirstDefForLinker->get();
1262}
1263
1264// The main entry point for creating the global ThinLTO analysis. The structure
1265// here is basically the same as before threads are spawned in the `run`
1266// function of `lib/LTO/ThinLTOCodeGenerator.cpp`.
1267extern "C" LLVMRustThinLTOData*
1268LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
1269                          int num_modules,
1270                          const char **preserved_symbols,
1271                          int num_symbols) {
1272  auto Ret = std::make_unique<LLVMRustThinLTOData>();
1273
1274  // Load each module's summary and merge it into one combined index
1275  for (int i = 0; i < num_modules; i++) {
1276    auto module = &modules[i];
1277    StringRef buffer(module->data, module->len);
1278    MemoryBufferRef mem_buffer(buffer, module->identifier);
1279
1280    Ret->ModuleMap[module->identifier] = mem_buffer;
1281
1282#if LLVM_VERSION_GE(18, 0)
1283    if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index)) {
1284#else
1285    if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) {
1286#endif
1287      LLVMRustSetLastError(toString(std::move(Err)).c_str());
1288      return nullptr;
1289    }
1290  }
1291
1292  // Collect for each module the list of function it defines (GUID -> Summary)
1293  Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries);
1294
1295  // Convert the preserved symbols set from string to GUID, this is then needed
1296  // for internalization.
1297  for (int i = 0; i < num_symbols; i++) {
1298    auto GUID = GlobalValue::getGUID(preserved_symbols[i]);
1299    Ret->GUIDPreservedSymbols.insert(GUID);
1300  }
1301
1302  // Collect the import/export lists for all modules from the call-graph in the
1303  // combined index
1304  //
1305  // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`
1306  auto deadIsPrevailing = [&](GlobalValue::GUID G) {
1307    return PrevailingType::Unknown;
1308  };
1309  // We don't have a complete picture in our use of ThinLTO, just our immediate
1310  // crate, so we need `ImportEnabled = false` to limit internalization.
1311  // Otherwise, we sometimes lose `static` values -- see #60184.
1312  computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols,
1313                                  deadIsPrevailing, /* ImportEnabled = */ false);
1314  // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it
1315  // impacts the caching.
1316  //
1317  // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this
1318  // being lifted from `lib/LTO/LTO.cpp` as well
1319  DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
1320  for (auto &I : Ret->Index) {
1321    if (I.second.SummaryList.size() > 1)
1322      PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList);
1323  }
1324  auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1325    const auto &Prevailing = PrevailingCopy.find(GUID);
1326    if (Prevailing == PrevailingCopy.end())
1327      return true;
1328    return Prevailing->second == S;
1329  };
1330  ComputeCrossModuleImport(
1331    Ret->Index,
1332    Ret->ModuleToDefinedGVSummaries,
1333#if LLVM_VERSION_GE(17, 0)
1334    isPrevailing,
1335#endif
1336    Ret->ImportLists,
1337    Ret->ExportLists
1338  );
1339
1340  auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1341                              GlobalValue::GUID GUID,
1342                              GlobalValue::LinkageTypes NewLinkage) {
1343    Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1344  };
1345
1346  // Uses FromPrevailing visibility scheme which works for many binary
1347  // formats. We probably could and should use ELF visibility scheme for many of
1348  // our targets, however.
1349  lto::Config conf;
1350  thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage,
1351                                  Ret->GUIDPreservedSymbols);
1352
1353  // Here we calculate an `ExportedGUIDs` set for use in the `isExported`
1354  // callback below. This callback below will dictate the linkage for all
1355  // summaries in the index, and we basically just only want to ensure that dead
1356  // symbols are internalized. Otherwise everything that's already external
1357  // linkage will stay as external, and internal will stay as internal.
1358  std::set<GlobalValue::GUID> ExportedGUIDs;
1359  for (auto &List : Ret->Index) {
1360    for (auto &GVS: List.second.SummaryList) {
1361      if (GlobalValue::isLocalLinkage(GVS->linkage()))
1362        continue;
1363      auto GUID = GVS->getOriginalName();
1364      if (GVS->flags().Live)
1365        ExportedGUIDs.insert(GUID);
1366    }
1367  }
1368  auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1369    const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier);
1370    return (ExportList != Ret->ExportLists.end() &&
1371      ExportList->second.count(VI)) ||
1372      ExportedGUIDs.count(VI.getGUID());
1373  };
1374  thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing);
1375
1376  return Ret.release();
1377}
1378
1379extern "C" void
1380LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
1381  delete Data;
1382}
1383
1384// Below are the various passes that happen *per module* when doing ThinLTO.
1385//
1386// In other words, these are the functions that are all run concurrently
1387// with one another, one per module. The passes here correspond to the analysis
1388// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the
1389// `ProcessThinLTOModule` function. Here they're split up into separate steps
1390// so rustc can save off the intermediate bytecode between each step.
1391
1392static bool
1393clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
1394  // When linking an ELF shared object, dso_local should be dropped. We
1395  // conservatively do this for -fpic.
1396  bool ClearDSOLocalOnDeclarations =
1397      TM.getTargetTriple().isOSBinFormatELF() &&
1398      TM.getRelocationModel() != Reloc::Static &&
1399      Mod.getPIELevel() == PIELevel::Default;
1400  return ClearDSOLocalOnDeclarations;
1401}
1402
1403extern "C" bool
1404LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1405                             LLVMTargetMachineRef TM) {
1406  Module &Mod = *unwrap(M);
1407  TargetMachine &Target = *unwrap(TM);
1408
1409  bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1410  bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
1411
1412  if (error) {
1413    LLVMRustSetLastError("renameModuleForThinLTO failed");
1414    return false;
1415  }
1416  return true;
1417}
1418
1419extern "C" bool
1420LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1421  Module &Mod = *unwrap(M);
1422  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1423  thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);
1424  return true;
1425}
1426
1427extern "C" bool
1428LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
1429  Module &Mod = *unwrap(M);
1430  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
1431  thinLTOInternalizeModule(Mod, DefinedGlobals);
1432  return true;
1433}
1434
1435extern "C" bool
1436LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
1437                             LLVMTargetMachineRef TM) {
1438  Module &Mod = *unwrap(M);
1439  TargetMachine &Target = *unwrap(TM);
1440
1441  const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
1442  auto Loader = [&](StringRef Identifier) {
1443    const auto &Memory = Data->ModuleMap.lookup(Identifier);
1444    auto &Context = Mod.getContext();
1445    auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true);
1446
1447    if (!MOrErr)
1448      return MOrErr;
1449
1450    // The rest of this closure is a workaround for
1451    // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports
1452    // we accidentally import wasm custom sections into different modules,
1453    // duplicating them by in the final output artifact.
1454    //
1455    // The issue is worked around here by manually removing the
1456    // `wasm.custom_sections` named metadata node from any imported module. This
1457    // we know isn't used by any optimization pass so there's no need for it to
1458    // be imported.
1459    //
1460    // Note that the metadata is currently lazily loaded, so we materialize it
1461    // here before looking up if there's metadata inside. The `FunctionImporter`
1462    // will immediately materialize metadata anyway after an import, so this
1463    // shouldn't be a perf hit.
1464    if (Error Err = (*MOrErr)->materializeMetadata()) {
1465      Expected<std::unique_ptr<Module>> Ret(std::move(Err));
1466      return Ret;
1467    }
1468
1469    auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections");
1470    if (WasmCustomSections)
1471      WasmCustomSections->eraseFromParent();
1472
1473    // `llvm.ident` named metadata also gets duplicated.
1474    auto *llvmIdent = (*MOrErr)->getNamedMetadata("llvm.ident");
1475    if (llvmIdent)
1476      llvmIdent->eraseFromParent();
1477
1478    return MOrErr;
1479  };
1480  bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
1481  FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
1482  Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
1483  if (!Result) {
1484    LLVMRustSetLastError(toString(Result.takeError()).c_str());
1485    return false;
1486  }
1487  return true;
1488}
1489
1490// This struct and various functions are sort of a hack right now, but the
1491// problem is that we've got in-memory LLVM modules after we generate and
1492// optimize all codegen-units for one compilation in rustc. To be compatible
1493// with the LTO support above we need to serialize the modules plus their
1494// ThinLTO summary into memory.
1495//
1496// This structure is basically an owned version of a serialize module, with
1497// a ThinLTO summary attached.
1498struct LLVMRustThinLTOBuffer {
1499  std::string data;
1500};
1501
1502extern "C" LLVMRustThinLTOBuffer*
1503LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) {
1504  auto Ret = std::make_unique<LLVMRustThinLTOBuffer>();
1505  {
1506    raw_string_ostream OS(Ret->data);
1507    {
1508      if (is_thin) {
1509        PassBuilder PB;
1510        LoopAnalysisManager LAM;
1511        FunctionAnalysisManager FAM;
1512        CGSCCAnalysisManager CGAM;
1513        ModuleAnalysisManager MAM;
1514        PB.registerModuleAnalyses(MAM);
1515        PB.registerCGSCCAnalyses(CGAM);
1516        PB.registerFunctionAnalyses(FAM);
1517        PB.registerLoopAnalyses(LAM);
1518        PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
1519        ModulePassManager MPM;
1520        MPM.addPass(ThinLTOBitcodeWriterPass(OS, nullptr));
1521        MPM.run(*unwrap(M), MAM);
1522      } else {
1523        WriteBitcodeToFile(*unwrap(M), OS);
1524      }
1525    }
1526  }
1527  return Ret.release();
1528}
1529
1530extern "C" void
1531LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) {
1532  delete Buffer;
1533}
1534
1535extern "C" const void*
1536LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) {
1537  return Buffer->data.data();
1538}
1539
1540extern "C" size_t
1541LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) {
1542  return Buffer->data.length();
1543}
1544
1545// This is what we used to parse upstream bitcode for actual ThinLTO
1546// processing. We'll call this once per module optimized through ThinLTO, and
1547// it'll be called concurrently on many threads.
1548extern "C" LLVMModuleRef
1549LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
1550                           const char *data,
1551                           size_t len,
1552                           const char *identifier) {
1553  StringRef Data(data, len);
1554  MemoryBufferRef Buffer(Data, identifier);
1555  unwrap(Context)->enableDebugTypeODRUniquing();
1556  Expected<std::unique_ptr<Module>> SrcOrError =
1557      parseBitcodeFile(Buffer, *unwrap(Context));
1558  if (!SrcOrError) {
1559    LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str());
1560    return nullptr;
1561  }
1562  return wrap(std::move(*SrcOrError).release());
1563}
1564
1565// Find the bitcode section in the object file data and return it as a slice.
1566// Fail if the bitcode section is present but empty.
1567//
1568// On success, the return value is the pointer to the start of the slice and
1569// `out_len` is filled with the (non-zero) length. On failure, the return value
1570// is `nullptr` and `out_len` is set to zero.
1571extern "C" const char*
1572LLVMRustGetBitcodeSliceFromObjectData(const char *data,
1573                                      size_t len,
1574                                      size_t *out_len) {
1575  *out_len = 0;
1576
1577  StringRef Data(data, len);
1578  MemoryBufferRef Buffer(Data, ""); // The id is unused.
1579
1580  Expected<MemoryBufferRef> BitcodeOrError =
1581    object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
1582  if (!BitcodeOrError) {
1583    LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
1584    return nullptr;
1585  }
1586
1587  *out_len = BitcodeOrError->getBufferSize();
1588  return BitcodeOrError->getBufferStart();
1589}
1590
1591// Find a section of an object file by name. Fail if the section is missing or
1592// empty.
1593extern "C" const char *LLVMRustGetSliceFromObjectDataByName(const char *data,
1594                                                            size_t len,
1595                                                            const char *name,
1596                                                            size_t *out_len) {
1597  *out_len = 0;
1598  StringRef Data(data, len);
1599  MemoryBufferRef Buffer(Data, ""); // The id is unused.
1600  file_magic Type = identify_magic(Buffer.getBuffer());
1601  Expected<std::unique_ptr<object::ObjectFile>> ObjFileOrError =
1602      object::ObjectFile::createObjectFile(Buffer, Type);
1603  if (!ObjFileOrError) {
1604    LLVMRustSetLastError(toString(ObjFileOrError.takeError()).c_str());
1605    return nullptr;
1606  }
1607  for (const object::SectionRef &Sec : (*ObjFileOrError)->sections()) {
1608    Expected<StringRef> Name = Sec.getName();
1609    if (Name && *Name == name) {
1610      Expected<StringRef> SectionOrError = Sec.getContents();
1611      if (!SectionOrError) {
1612        LLVMRustSetLastError(toString(SectionOrError.takeError()).c_str());
1613        return nullptr;
1614      }
1615      *out_len = SectionOrError->size();
1616      return SectionOrError->data();
1617    }
1618  }
1619  LLVMRustSetLastError("could not find requested section");
1620  return nullptr;
1621}
1622
1623// Computes the LTO cache key for the provided 'ModId' in the given 'Data',
1624// storing the result in 'KeyOut'.
1625// Currently, this cache key is a SHA-1 hash of anything that could affect
1626// the result of optimizing this module (e.g. module imports, exports, liveness
1627// of access globals, etc).
1628// The precise details are determined by LLVM in `computeLTOCacheKey`, which is
1629// used during the normal linker-plugin incremental thin-LTO process.
1630extern "C" void
1631LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) {
1632  SmallString<40> Key;
1633  llvm::lto::Config conf;
1634  const auto &ImportList = Data->ImportLists.lookup(ModId);
1635  const auto &ExportList = Data->ExportLists.lookup(ModId);
1636  const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId);
1637  const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId);
1638  std::set<GlobalValue::GUID> CfiFunctionDefs;
1639  std::set<GlobalValue::GUID> CfiFunctionDecls;
1640
1641  // Based on the 'InProcessThinBackend' constructor in LLVM
1642  for (auto &Name : Data->Index.cfiFunctionDefs())
1643    CfiFunctionDefs.insert(
1644        GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1645  for (auto &Name : Data->Index.cfiFunctionDecls())
1646    CfiFunctionDecls.insert(
1647        GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
1648
1649  llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId,
1650      ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls
1651  );
1652
1653  LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size());
1654}