fork/rust
git clone https://git.julia.blue/fork/rust
3bcad65fbf6
master
1#include <stdio.h> 2 3#include <cstddef> 4#include <iomanip> 5#include <vector> 6#include <set> 7 8#include "LLVMWrapper.h" 9 10#include "llvm/Analysis/AliasAnalysis.h" 11#include "llvm/Analysis/TargetLibraryInfo.h" 12#include "llvm/Analysis/TargetTransformInfo.h" 13#include "llvm/CodeGen/CommandFlags.h" 14#include "llvm/CodeGen/TargetSubtargetInfo.h" 15#include "llvm/IR/AutoUpgrade.h" 16#include "llvm/IR/AssemblyAnnotationWriter.h" 17#include "llvm/IR/IntrinsicInst.h" 18#include "llvm/IR/Verifier.h" 19#include "llvm/MC/TargetRegistry.h" 20#include "llvm/Object/ObjectFile.h" 21#include "llvm/Object/IRObjectFile.h" 22#include "llvm/Passes/PassBuilder.h" 23#include "llvm/Passes/PassPlugin.h" 24#include "llvm/Passes/StandardInstrumentations.h" 25#include "llvm/Support/CBindingWrapping.h" 26#include "llvm/Support/FileSystem.h" 27#if LLVM_VERSION_GE(17, 0) 28#include "llvm/Support/VirtualFileSystem.h" 29#endif 30#include "llvm/Target/TargetMachine.h" 31#include "llvm/Transforms/IPO/AlwaysInliner.h" 32#include "llvm/Transforms/IPO/FunctionImport.h" 33#include "llvm/Transforms/IPO/Internalize.h" 34#include "llvm/Transforms/IPO/LowerTypeTests.h" 35#include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h" 36#include "llvm/Transforms/Utils/AddDiscriminators.h" 37#include "llvm/Transforms/Utils/FunctionImportUtils.h" 38#include "llvm/LTO/LTO.h" 39#include "llvm/Bitcode/BitcodeWriter.h" 40 41#include "llvm/Transforms/Instrumentation.h" 42#include "llvm/Transforms/Instrumentation/AddressSanitizer.h" 43#include "llvm/Support/TimeProfiler.h" 44#include "llvm/Transforms/Instrumentation/GCOVProfiler.h" 45#include "llvm/Transforms/Instrumentation/InstrProfiling.h" 46#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" 47#include "llvm/Transforms/Instrumentation/MemorySanitizer.h" 48#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" 49#include "llvm/Transforms/Utils/CanonicalizeAliases.h" 50#include "llvm/Transforms/Utils/NameAnonGlobals.h" 51#include "llvm/Transforms/Utils.h" 52 53using namespace llvm; 54 55static codegen::RegisterCodeGenFlags CGF; 56 57typedef struct LLVMOpaquePass *LLVMPassRef; 58typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; 59 60DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef) 61DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) 62 63extern "C" void LLVMTimeTraceProfilerInitialize() { 64 timeTraceProfilerInitialize( 65 /* TimeTraceGranularity */ 0, 66 /* ProcName */ "rustc"); 67} 68 69extern "C" void LLVMTimeTraceProfilerFinishThread() { 70 timeTraceProfilerFinishThread(); 71} 72 73extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) { 74 StringRef FN(FileName); 75 std::error_code EC; 76 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways); 77 78 timeTraceProfilerWrite(OS); 79 timeTraceProfilerCleanup(); 80} 81 82#ifdef LLVM_COMPONENT_X86 83#define SUBTARGET_X86 SUBTARGET(X86) 84#else 85#define SUBTARGET_X86 86#endif 87 88#ifdef LLVM_COMPONENT_ARM 89#define SUBTARGET_ARM SUBTARGET(ARM) 90#else 91#define SUBTARGET_ARM 92#endif 93 94#ifdef LLVM_COMPONENT_AARCH64 95#define SUBTARGET_AARCH64 SUBTARGET(AArch64) 96#else 97#define SUBTARGET_AARCH64 98#endif 99 100#ifdef LLVM_COMPONENT_AVR 101#define SUBTARGET_AVR SUBTARGET(AVR) 102#else 103#define SUBTARGET_AVR 104#endif 105 106#ifdef LLVM_COMPONENT_M68k 107#define SUBTARGET_M68K SUBTARGET(M68k) 108#else 109#define SUBTARGET_M68K 110#endif 111 112#ifdef LLVM_COMPONENT_CSKY 113#define SUBTARGET_CSKY SUBTARGET(CSKY) 114#else 115#define SUBTARGET_CSKY 116#endif 117 118#ifdef LLVM_COMPONENT_MIPS 119#define SUBTARGET_MIPS SUBTARGET(Mips) 120#else 121#define SUBTARGET_MIPS 122#endif 123 124#ifdef LLVM_COMPONENT_POWERPC 125#define SUBTARGET_PPC SUBTARGET(PPC) 126#else 127#define SUBTARGET_PPC 128#endif 129 130#ifdef LLVM_COMPONENT_SYSTEMZ 131#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ) 132#else 133#define SUBTARGET_SYSTEMZ 134#endif 135 136#ifdef LLVM_COMPONENT_MSP430 137#define SUBTARGET_MSP430 SUBTARGET(MSP430) 138#else 139#define SUBTARGET_MSP430 140#endif 141 142#ifdef LLVM_COMPONENT_RISCV 143#define SUBTARGET_RISCV SUBTARGET(RISCV) 144#else 145#define SUBTARGET_RISCV 146#endif 147 148#ifdef LLVM_COMPONENT_SPARC 149#define SUBTARGET_SPARC SUBTARGET(Sparc) 150#else 151#define SUBTARGET_SPARC 152#endif 153 154#ifdef LLVM_COMPONENT_HEXAGON 155#define SUBTARGET_HEXAGON SUBTARGET(Hexagon) 156#else 157#define SUBTARGET_HEXAGON 158#endif 159 160#ifdef LLVM_COMPONENT_LOONGARCH 161#define SUBTARGET_LOONGARCH SUBTARGET(LoongArch) 162#else 163#define SUBTARGET_LOONGARCH 164#endif 165 166#define GEN_SUBTARGETS \ 167 SUBTARGET_X86 \ 168 SUBTARGET_ARM \ 169 SUBTARGET_AARCH64 \ 170 SUBTARGET_AVR \ 171 SUBTARGET_M68K \ 172 SUBTARGET_CSKY \ 173 SUBTARGET_MIPS \ 174 SUBTARGET_PPC \ 175 SUBTARGET_SYSTEMZ \ 176 SUBTARGET_MSP430 \ 177 SUBTARGET_SPARC \ 178 SUBTARGET_HEXAGON \ 179 SUBTARGET_RISCV \ 180 SUBTARGET_LOONGARCH \ 181 182#define SUBTARGET(x) \ 183 namespace llvm { \ 184 extern const SubtargetFeatureKV x##FeatureKV[]; \ 185 extern const SubtargetFeatureKV x##SubTypeKV[]; \ 186 } 187 188GEN_SUBTARGETS 189#undef SUBTARGET 190 191extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, 192 const char *Feature) { 193 TargetMachine *Target = unwrap(TM); 194 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 195 return MCInfo->checkFeatures(std::string("+") + Feature); 196} 197 198enum class LLVMRustCodeModel { 199 Tiny, 200 Small, 201 Kernel, 202 Medium, 203 Large, 204 None, 205}; 206 207#if LLVM_VERSION_LT(16, 0) 208static Optional<CodeModel::Model> 209#else 210static std::optional<CodeModel::Model> 211#endif 212fromRust(LLVMRustCodeModel Model) { 213 switch (Model) { 214 case LLVMRustCodeModel::Tiny: 215 return CodeModel::Tiny; 216 case LLVMRustCodeModel::Small: 217 return CodeModel::Small; 218 case LLVMRustCodeModel::Kernel: 219 return CodeModel::Kernel; 220 case LLVMRustCodeModel::Medium: 221 return CodeModel::Medium; 222 case LLVMRustCodeModel::Large: 223 return CodeModel::Large; 224 case LLVMRustCodeModel::None: 225#if LLVM_VERSION_LT(16, 0) 226 return None; 227#else 228 return std::nullopt; 229#endif 230 default: 231 report_fatal_error("Bad CodeModel."); 232 } 233} 234 235enum class LLVMRustCodeGenOptLevel { 236 None, 237 Less, 238 Default, 239 Aggressive, 240}; 241 242#if LLVM_VERSION_GE(18, 0) 243 using CodeGenOptLevelEnum = llvm::CodeGenOptLevel; 244#else 245 using CodeGenOptLevelEnum = llvm::CodeGenOpt::Level; 246#endif 247 248static CodeGenOptLevelEnum fromRust(LLVMRustCodeGenOptLevel Level) { 249 switch (Level) { 250 case LLVMRustCodeGenOptLevel::None: 251 return CodeGenOptLevelEnum::None; 252 case LLVMRustCodeGenOptLevel::Less: 253 return CodeGenOptLevelEnum::Less; 254 case LLVMRustCodeGenOptLevel::Default: 255 return CodeGenOptLevelEnum::Default; 256 case LLVMRustCodeGenOptLevel::Aggressive: 257 return CodeGenOptLevelEnum::Aggressive; 258 default: 259 report_fatal_error("Bad CodeGenOptLevel."); 260 } 261} 262 263enum class LLVMRustPassBuilderOptLevel { 264 O0, 265 O1, 266 O2, 267 O3, 268 Os, 269 Oz, 270}; 271 272static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) { 273 switch (Level) { 274 case LLVMRustPassBuilderOptLevel::O0: 275 return OptimizationLevel::O0; 276 case LLVMRustPassBuilderOptLevel::O1: 277 return OptimizationLevel::O1; 278 case LLVMRustPassBuilderOptLevel::O2: 279 return OptimizationLevel::O2; 280 case LLVMRustPassBuilderOptLevel::O3: 281 return OptimizationLevel::O3; 282 case LLVMRustPassBuilderOptLevel::Os: 283 return OptimizationLevel::Os; 284 case LLVMRustPassBuilderOptLevel::Oz: 285 return OptimizationLevel::Oz; 286 default: 287 report_fatal_error("Bad PassBuilderOptLevel."); 288 } 289} 290 291enum class LLVMRustRelocModel { 292 Static, 293 PIC, 294 DynamicNoPic, 295 ROPI, 296 RWPI, 297 ROPIRWPI, 298}; 299 300static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) { 301 switch (RustReloc) { 302 case LLVMRustRelocModel::Static: 303 return Reloc::Static; 304 case LLVMRustRelocModel::PIC: 305 return Reloc::PIC_; 306 case LLVMRustRelocModel::DynamicNoPic: 307 return Reloc::DynamicNoPIC; 308 case LLVMRustRelocModel::ROPI: 309 return Reloc::ROPI; 310 case LLVMRustRelocModel::RWPI: 311 return Reloc::RWPI; 312 case LLVMRustRelocModel::ROPIRWPI: 313 return Reloc::ROPI_RWPI; 314 } 315 report_fatal_error("Bad RelocModel."); 316} 317 318/// getLongestEntryLength - Return the length of the longest entry in the table. 319template<typename KV> 320static size_t getLongestEntryLength(ArrayRef<KV> Table) { 321 size_t MaxLen = 0; 322 for (auto &I : Table) 323 MaxLen = std::max(MaxLen, std::strlen(I.Key)); 324 return MaxLen; 325} 326 327using PrintBackendInfo = void(void*, const char* Data, size_t Len); 328 329extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM, 330 const char* TargetCPU, 331 PrintBackendInfo Print, 332 void* Out) { 333 const TargetMachine *Target = unwrap(TM); 334 const Triple::ArchType HostArch = Triple(sys::getDefaultTargetTriple()).getArch(); 335 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); 336 337 std::ostringstream Buf; 338 339#if LLVM_VERSION_GE(17, 0) 340 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 341 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getAllProcessorDescriptions(); 342#else 343 Buf << "Full target CPU help is not supported by this LLVM version.\n\n"; 344 SubtargetSubTypeKV TargetCPUKV = { TargetCPU, {{}}, {{}} }; 345 const ArrayRef<SubtargetSubTypeKV> CPUTable = TargetCPUKV; 346#endif 347 unsigned MaxCPULen = getLongestEntryLength(CPUTable); 348 349 Buf << "Available CPUs for this target:\n"; 350 // Don't print the "native" entry when the user specifies --target with a 351 // different arch since that could be wrong or misleading. 352 if (HostArch == TargetArch) { 353 MaxCPULen = std::max(MaxCPULen, (unsigned) std::strlen("native")); 354 const StringRef HostCPU = sys::getHostCPUName(); 355 Buf << " " << std::left << std::setw(MaxCPULen) << "native" 356 << " - Select the CPU of the current host " 357 "(currently " << HostCPU.str() << ").\n"; 358 } 359 for (auto &CPU : CPUTable) { 360 // Compare cpu against current target to label the default 361 if (strcmp(CPU.Key, TargetCPU) == 0) { 362 Buf << " " << std::left << std::setw(MaxCPULen) << CPU.Key 363 << " - This is the default target CPU for the current build target " 364 "(currently " << Target->getTargetTriple().str() << ")."; 365 } 366 else { 367 Buf << " " << CPU.Key; 368 } 369 Buf << "\n"; 370 } 371 372 const auto &BufString = Buf.str(); 373 Print(Out, BufString.data(), BufString.size()); 374} 375 376extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) { 377#ifdef LLVM_RUSTLLVM 378 const TargetMachine *Target = unwrap(TM); 379 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 380 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable(); 381 return FeatTable.size(); 382#else 383 return 0; 384#endif 385} 386 387extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index, 388 const char** Feature, const char** Desc) { 389#ifdef LLVM_RUSTLLVM 390 const TargetMachine *Target = unwrap(TM); 391 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 392 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable(); 393 const SubtargetFeatureKV Feat = FeatTable[Index]; 394 *Feature = Feat.Key; 395 *Desc = Feat.Desc; 396#endif 397} 398 399extern "C" const char* LLVMRustGetHostCPUName(size_t *len) { 400 StringRef Name = sys::getHostCPUName(); 401 *len = Name.size(); 402 return Name.data(); 403} 404 405extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( 406 const char *TripleStr, const char *CPU, const char *Feature, 407 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc, 408 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, 409 bool FunctionSections, 410 bool DataSections, 411 bool UniqueSectionNames, 412 bool TrapUnreachable, 413 bool Singlethread, 414 bool AsmComments, 415 bool EmitStackSizeSection, 416 bool RelaxELFRelocations, 417 bool UseInitArray, 418 const char *SplitDwarfFile, 419 const char *OutputObjFile, 420 const char *DebugInfoCompression, 421 bool ForceEmulatedTls, 422 const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) { 423 424 auto OptLevel = fromRust(RustOptLevel); 425 auto RM = fromRust(RustReloc); 426 auto CM = fromRust(RustCM); 427 428 std::string Error; 429 Triple Trip(Triple::normalize(TripleStr)); 430 const llvm::Target *TheTarget = 431 TargetRegistry::lookupTarget(Trip.getTriple(), Error); 432 if (TheTarget == nullptr) { 433 LLVMRustSetLastError(Error.c_str()); 434 return nullptr; 435 } 436 437 TargetOptions Options = codegen::InitTargetOptionsFromCodeGenFlags(Trip); 438 439 Options.FloatABIType = FloatABI::Default; 440 if (UseSoftFloat) { 441 Options.FloatABIType = FloatABI::Soft; 442 } 443 Options.DataSections = DataSections; 444 Options.FunctionSections = FunctionSections; 445 Options.UniqueSectionNames = UniqueSectionNames; 446 Options.MCOptions.AsmVerbose = AsmComments; 447 Options.MCOptions.PreserveAsmComments = AsmComments; 448 Options.MCOptions.ABIName = ABIStr; 449 if (SplitDwarfFile) { 450 Options.MCOptions.SplitDwarfFile = SplitDwarfFile; 451 } 452 if (OutputObjFile) { 453 Options.ObjectFilenameForDebug = OutputObjFile; 454 } 455#if LLVM_VERSION_GE(16, 0) 456 if (!strcmp("zlib", DebugInfoCompression) && llvm::compression::zlib::isAvailable()) { 457 Options.CompressDebugSections = DebugCompressionType::Zlib; 458 } else if (!strcmp("zstd", DebugInfoCompression) && llvm::compression::zstd::isAvailable()) { 459 Options.CompressDebugSections = DebugCompressionType::Zstd; 460 } else if (!strcmp("none", DebugInfoCompression)) { 461 Options.CompressDebugSections = DebugCompressionType::None; 462 } 463#endif 464 465 Options.RelaxELFRelocations = RelaxELFRelocations; 466 Options.UseInitArray = UseInitArray; 467 468#if LLVM_VERSION_LT(17, 0) 469 if (ForceEmulatedTls) { 470 Options.ExplicitEmulatedTLS = true; 471 Options.EmulatedTLS = true; 472 } 473#else 474 Options.EmulatedTLS = ForceEmulatedTls || Trip.hasDefaultEmulatedTLS(); 475#endif 476 477 if (TrapUnreachable) { 478 // Tell LLVM to codegen `unreachable` into an explicit trap instruction. 479 // This limits the extent of possible undefined behavior in some cases, as 480 // it prevents control flow from "falling through" into whatever code 481 // happens to be laid out next in memory. 482 Options.TrapUnreachable = true; 483 } 484 485 if (Singlethread) { 486 Options.ThreadModel = ThreadModel::Single; 487 } 488 489 Options.EmitStackSizeSection = EmitStackSizeSection; 490 491 492 if (ArgsCstrBuff != nullptr) 493 { 494 int buffer_offset = 0; 495 assert(ArgsCstrBuff[ArgsCstrBuffLen - 1] == '\0'); 496 497 const size_t arg0_len = std::strlen(ArgsCstrBuff); 498 char* arg0 = new char[arg0_len + 1]; 499 memcpy(arg0, ArgsCstrBuff, arg0_len); 500 arg0[arg0_len] = '\0'; 501 buffer_offset += arg0_len + 1; 502 503 const int num_cmd_arg_strings = 504 std::count(&ArgsCstrBuff[buffer_offset], &ArgsCstrBuff[ArgsCstrBuffLen], '\0'); 505 506 std::string* cmd_arg_strings = new std::string[num_cmd_arg_strings]; 507 for (int i = 0; i < num_cmd_arg_strings; ++i) 508 { 509 assert(buffer_offset < ArgsCstrBuffLen); 510 const int len = std::strlen(ArgsCstrBuff + buffer_offset); 511 cmd_arg_strings[i] = std::string(&ArgsCstrBuff[buffer_offset], len); 512 buffer_offset += len + 1; 513 } 514 515 assert(buffer_offset == ArgsCstrBuffLen); 516 517 Options.MCOptions.Argv0 = arg0; 518 Options.MCOptions.CommandLineArgs = 519 llvm::ArrayRef<std::string>(cmd_arg_strings, num_cmd_arg_strings); 520 } 521 522 TargetMachine *TM = TheTarget->createTargetMachine( 523 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); 524 return wrap(TM); 525} 526 527extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { 528 529 MCTargetOptions& MCOptions = unwrap(TM)->Options.MCOptions; 530 delete[] MCOptions.Argv0; 531 delete[] MCOptions.CommandLineArgs.data(); 532 533 delete unwrap(TM); 534} 535 536// Unfortunately, the LLVM C API doesn't provide a way to create the 537// TargetLibraryInfo pass, so we use this method to do so. 538extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M, 539 bool DisableSimplifyLibCalls) { 540 Triple TargetTriple(unwrap(M)->getTargetTriple()); 541 TargetLibraryInfoImpl TLII(TargetTriple); 542 if (DisableSimplifyLibCalls) 543 TLII.disableAllFunctions(); 544 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII)); 545} 546 547extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) { 548 // Initializing the command-line options more than once is not allowed. So, 549 // check if they've already been initialized. (This could happen if we're 550 // being called from rustpkg, for example). If the arguments change, then 551 // that's just kinda unfortunate. 552 static bool Initialized = false; 553 if (Initialized) 554 return; 555 Initialized = true; 556 cl::ParseCommandLineOptions(Argc, Argv); 557} 558 559enum class LLVMRustFileType { 560 AssemblyFile, 561 ObjectFile, 562}; 563 564static CodeGenFileType fromRust(LLVMRustFileType Type) { 565 switch (Type) { 566 case LLVMRustFileType::AssemblyFile: 567#if LLVM_VERSION_GE(18, 0) 568 return CodeGenFileType::AssemblyFile; 569#else 570 return CGFT_AssemblyFile; 571#endif 572 case LLVMRustFileType::ObjectFile: 573#if LLVM_VERSION_GE(18, 0) 574 return CodeGenFileType::ObjectFile; 575#else 576 return CGFT_ObjectFile; 577#endif 578 default: 579 report_fatal_error("Bad FileType."); 580 } 581} 582 583extern "C" LLVMRustResult 584LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, 585 LLVMModuleRef M, const char *Path, const char *DwoPath, 586 LLVMRustFileType RustFileType) { 587 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR); 588 auto FileType = fromRust(RustFileType); 589 590 std::string ErrorInfo; 591 std::error_code EC; 592 raw_fd_ostream OS(Path, EC, sys::fs::OF_None); 593 if (EC) 594 ErrorInfo = EC.message(); 595 if (ErrorInfo != "") { 596 LLVMRustSetLastError(ErrorInfo.c_str()); 597 return LLVMRustResult::Failure; 598 } 599 600 buffer_ostream BOS(OS); 601 if (DwoPath) { 602 raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None); 603 EC.clear(); 604 if (EC) 605 ErrorInfo = EC.message(); 606 if (ErrorInfo != "") { 607 LLVMRustSetLastError(ErrorInfo.c_str()); 608 return LLVMRustResult::Failure; 609 } 610 buffer_ostream DBOS(DOS); 611 unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false); 612 PM->run(*unwrap(M)); 613 } else { 614 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false); 615 PM->run(*unwrap(M)); 616 } 617 618 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output 619 // stream (OS), so the only real safe place to delete this is here? Don't we 620 // wish this was written in Rust? 621 LLVMDisposePassManager(PMR); 622 return LLVMRustResult::Success; 623} 624 625extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler 626 const char*, // pass name 627 const char*); // IR name 628extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler 629 630std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { 631 if (const auto *Cast = any_cast<const Module *>(&WrappedIr)) 632 return (*Cast)->getName().str(); 633 if (const auto *Cast = any_cast<const Function *>(&WrappedIr)) 634 return (*Cast)->getName().str(); 635 if (const auto *Cast = any_cast<const Loop *>(&WrappedIr)) 636 return (*Cast)->getName().str(); 637 if (const auto *Cast = any_cast<const LazyCallGraph::SCC *>(&WrappedIr)) 638 return (*Cast)->getName(); 639 return "<UNKNOWN>"; 640} 641 642 643void LLVMSelfProfileInitializeCallbacks( 644 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler, 645 LLVMRustSelfProfileBeforePassCallback BeforePassCallback, 646 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { 647 PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback]( 648 StringRef Pass, llvm::Any Ir) { 649 std::string PassName = Pass.str(); 650 std::string IrName = LLVMRustwrappedIrGetName(Ir); 651 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); 652 }); 653 654 PIC.registerAfterPassCallback( 655 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR, 656 const PreservedAnalyses &Preserved) { 657 AfterPassCallback(LlvmSelfProfiler); 658 }); 659 660 PIC.registerAfterPassInvalidatedCallback( 661 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) { 662 AfterPassCallback(LlvmSelfProfiler); 663 }); 664 665 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback]( 666 StringRef Pass, llvm::Any Ir) { 667 std::string PassName = Pass.str(); 668 std::string IrName = LLVMRustwrappedIrGetName(Ir); 669 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); 670 }); 671 672 PIC.registerAfterAnalysisCallback( 673 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { 674 AfterPassCallback(LlvmSelfProfiler); 675 }); 676} 677 678enum class LLVMRustOptStage { 679 PreLinkNoLTO, 680 PreLinkThinLTO, 681 PreLinkFatLTO, 682 ThinLTO, 683 FatLTO, 684}; 685 686struct LLVMRustSanitizerOptions { 687 bool SanitizeAddress; 688 bool SanitizeAddressRecover; 689 bool SanitizeCFI; 690 bool SanitizeKCFI; 691 bool SanitizeMemory; 692 bool SanitizeMemoryRecover; 693 int SanitizeMemoryTrackOrigins; 694 bool SanitizeThread; 695 bool SanitizeHWAddress; 696 bool SanitizeHWAddressRecover; 697 bool SanitizeKernelAddress; 698 bool SanitizeKernelAddressRecover; 699}; 700 701extern "C" LLVMRustResult 702LLVMRustOptimize( 703 LLVMModuleRef ModuleRef, 704 LLVMTargetMachineRef TMRef, 705 LLVMRustPassBuilderOptLevel OptLevelRust, 706 LLVMRustOptStage OptStage, 707 bool IsLinkerPluginLTO, 708 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers, 709 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, 710 bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers, 711 LLVMRustSanitizerOptions *SanitizerOptions, 712 const char *PGOGenPath, const char *PGOUsePath, 713 bool InstrumentCoverage, const char *InstrProfileOutput, 714 bool InstrumentGCOV, 715 const char *PGOSampleUsePath, bool DebugInfoForProfiling, 716 void* LlvmSelfProfiler, 717 LLVMRustSelfProfileBeforePassCallback BeforePassCallback, 718 LLVMRustSelfProfileAfterPassCallback AfterPassCallback, 719 const char *ExtraPasses, size_t ExtraPassesLen, 720 const char *LLVMPlugins, size_t LLVMPluginsLen) { 721 Module *TheModule = unwrap(ModuleRef); 722 TargetMachine *TM = unwrap(TMRef); 723 OptimizationLevel OptLevel = fromRust(OptLevelRust); 724 725 726 PipelineTuningOptions PTO; 727 PTO.LoopUnrolling = UnrollLoops; 728 PTO.LoopInterleaving = UnrollLoops; 729 PTO.LoopVectorization = LoopVectorize; 730 PTO.SLPVectorization = SLPVectorize; 731 PTO.MergeFunctions = MergeFunctions; 732 733 // FIXME: We may want to expose this as an option. 734 bool DebugPassManager = false; 735 736 PassInstrumentationCallbacks PIC; 737#if LLVM_VERSION_LT(16, 0) 738 StandardInstrumentations SI(DebugPassManager); 739#else 740 StandardInstrumentations SI(TheModule->getContext(), DebugPassManager); 741#endif 742 SI.registerCallbacks(PIC); 743 744 if (LlvmSelfProfiler){ 745 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback); 746 } 747 748#if LLVM_VERSION_LT(16, 0) 749 Optional<PGOOptions> PGOOpt; 750#else 751 std::optional<PGOOptions> PGOOpt; 752#endif 753#if LLVM_VERSION_GE(17, 0) 754 auto FS = vfs::getRealFileSystem(); 755#endif 756 if (PGOGenPath) { 757 assert(!PGOUsePath && !PGOSampleUsePath); 758 PGOOpt = PGOOptions(PGOGenPath, "", "", 759#if LLVM_VERSION_GE(17, 0) 760 "", 761 FS, 762#endif 763 PGOOptions::IRInstr, PGOOptions::NoCSAction, 764 DebugInfoForProfiling); 765 } else if (PGOUsePath) { 766 assert(!PGOSampleUsePath); 767 PGOOpt = PGOOptions(PGOUsePath, "", "", 768#if LLVM_VERSION_GE(17, 0) 769 "", 770 FS, 771#endif 772 PGOOptions::IRUse, PGOOptions::NoCSAction, 773 DebugInfoForProfiling); 774 } else if (PGOSampleUsePath) { 775 PGOOpt = PGOOptions(PGOSampleUsePath, "", "", 776#if LLVM_VERSION_GE(17, 0) 777 "", 778 FS, 779#endif 780 PGOOptions::SampleUse, PGOOptions::NoCSAction, 781 DebugInfoForProfiling); 782 } else if (DebugInfoForProfiling) { 783 PGOOpt = PGOOptions("", "", "", 784#if LLVM_VERSION_GE(17, 0) 785 "", 786 FS, 787#endif 788 PGOOptions::NoAction, PGOOptions::NoCSAction, 789 DebugInfoForProfiling); 790 } 791 792 PassBuilder PB(TM, PTO, PGOOpt, &PIC); 793 LoopAnalysisManager LAM; 794 FunctionAnalysisManager FAM; 795 CGSCCAnalysisManager CGAM; 796 ModuleAnalysisManager MAM; 797 798 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); 799 800 Triple TargetTriple(TheModule->getTargetTriple()); 801 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple)); 802 if (DisableSimplifyLibCalls) 803 TLII->disableAllFunctions(); 804 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); 805 806 PB.registerModuleAnalyses(MAM); 807 PB.registerCGSCCAnalyses(CGAM); 808 PB.registerFunctionAnalyses(FAM); 809 PB.registerLoopAnalyses(LAM); 810 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); 811 812 // We manually collect pipeline callbacks so we can apply them at O0, where the 813 // PassBuilder does not create a pipeline. 814 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>> 815 PipelineStartEPCallbacks; 816 std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>> 817 OptimizerLastEPCallbacks; 818 819 if (!IsLinkerPluginLTO 820 && SanitizerOptions && SanitizerOptions->SanitizeCFI 821 && !NoPrepopulatePasses) { 822 PipelineStartEPCallbacks.push_back( 823 [](ModulePassManager &MPM, OptimizationLevel Level) { 824 MPM.addPass(LowerTypeTestsPass(/*ExportSummary=*/nullptr, 825 /*ImportSummary=*/nullptr, 826 /*DropTypeTests=*/false)); 827 } 828 ); 829 } 830 831 if (VerifyIR) { 832 PipelineStartEPCallbacks.push_back( 833 [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) { 834 MPM.addPass(VerifierPass()); 835 } 836 ); 837 } 838 839 if (InstrumentGCOV) { 840 PipelineStartEPCallbacks.push_back( 841 [](ModulePassManager &MPM, OptimizationLevel Level) { 842 MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault())); 843 } 844 ); 845 } 846 847 if (InstrumentCoverage) { 848 PipelineStartEPCallbacks.push_back( 849 [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) { 850 InstrProfOptions Options; 851 if (InstrProfileOutput) { 852 Options.InstrProfileOutput = InstrProfileOutput; 853 } 854 // cargo run tests in multhreading mode by default 855 // so use atomics for coverage counters 856 Options.Atomic = true; 857 MPM.addPass(InstrProfiling(Options, false)); 858 } 859 ); 860 } 861 862 if (SanitizerOptions) { 863 if (SanitizerOptions->SanitizeMemory) { 864 MemorySanitizerOptions Options( 865 SanitizerOptions->SanitizeMemoryTrackOrigins, 866 SanitizerOptions->SanitizeMemoryRecover, 867 /*CompileKernel=*/false, 868 /*EagerChecks=*/true); 869 OptimizerLastEPCallbacks.push_back( 870 [Options](ModulePassManager &MPM, OptimizationLevel Level) { 871#if LLVM_VERSION_LT(16, 0) 872 MPM.addPass(ModuleMemorySanitizerPass(Options)); 873 MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options))); 874#else 875 MPM.addPass(MemorySanitizerPass(Options)); 876#endif 877 } 878 ); 879 } 880 881 if (SanitizerOptions->SanitizeThread) { 882 OptimizerLastEPCallbacks.push_back( 883 [](ModulePassManager &MPM, OptimizationLevel Level) { 884 MPM.addPass(ModuleThreadSanitizerPass()); 885 MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); 886 } 887 ); 888 } 889 890 if (SanitizerOptions->SanitizeAddress || SanitizerOptions->SanitizeKernelAddress) { 891 OptimizerLastEPCallbacks.push_back( 892 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { 893 auto CompileKernel = SanitizerOptions->SanitizeKernelAddress; 894 AddressSanitizerOptions opts = AddressSanitizerOptions{ 895 CompileKernel, 896 SanitizerOptions->SanitizeAddressRecover 897 || SanitizerOptions->SanitizeKernelAddressRecover, 898 /*UseAfterScope=*/true, 899 AsanDetectStackUseAfterReturnMode::Runtime, 900 }; 901#if LLVM_VERSION_LT(16, 0) 902 MPM.addPass(ModuleAddressSanitizerPass(opts)); 903#else 904 MPM.addPass(AddressSanitizerPass(opts)); 905#endif 906 } 907 ); 908 } 909 if (SanitizerOptions->SanitizeHWAddress) { 910 OptimizerLastEPCallbacks.push_back( 911 [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { 912 HWAddressSanitizerOptions opts( 913 /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover, 914 /*DisableOptimization=*/false); 915 MPM.addPass(HWAddressSanitizerPass(opts)); 916 } 917 ); 918 } 919 } 920 921 if (LLVMPluginsLen) { 922 auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen); 923 SmallVector<StringRef> Plugins; 924 PluginsStr.split(Plugins, ',', -1, false); 925 for (auto PluginPath: Plugins) { 926 auto Plugin = PassPlugin::Load(PluginPath.str()); 927 if (!Plugin) { 928 LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str()); 929 return LLVMRustResult::Failure; 930 } 931 Plugin->registerPassBuilderCallbacks(PB); 932 } 933 } 934 935 ModulePassManager MPM; 936 bool NeedThinLTOBufferPasses = UseThinLTOBuffers; 937 if (!NoPrepopulatePasses) { 938 // The pre-link pipelines don't support O0 and require using buildO0DefaultPipeline() instead. 939 // At the same time, the LTO pipelines do support O0 and using them is required. 940 bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO; 941 if (OptLevel == OptimizationLevel::O0 && !IsLTO) { 942 for (const auto &C : PipelineStartEPCallbacks) 943 PB.registerPipelineStartEPCallback(C); 944 for (const auto &C : OptimizerLastEPCallbacks) 945 PB.registerOptimizerLastEPCallback(C); 946 947 // Pass false as we manually schedule ThinLTOBufferPasses below. 948 MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false); 949 } else { 950 for (const auto &C : PipelineStartEPCallbacks) 951 PB.registerPipelineStartEPCallback(C); 952 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) { 953 for (const auto &C : OptimizerLastEPCallbacks) 954 PB.registerOptimizerLastEPCallback(C); 955 } 956 957 switch (OptStage) { 958 case LLVMRustOptStage::PreLinkNoLTO: 959 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager); 960 break; 961 case LLVMRustOptStage::PreLinkThinLTO: 962 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel); 963 // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback 964 // passes may still run afterwards. This means we need to run the buffer passes again. 965 // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks 966 // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks. 967 if (OptimizerLastEPCallbacks.empty()) 968 NeedThinLTOBufferPasses = false; 969 for (const auto &C : OptimizerLastEPCallbacks) 970 C(MPM, OptLevel); 971 break; 972 case LLVMRustOptStage::PreLinkFatLTO: 973 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel); 974 NeedThinLTOBufferPasses = false; 975 break; 976 case LLVMRustOptStage::ThinLTO: 977 // FIXME: Does it make sense to pass the ModuleSummaryIndex? 978 // It only seems to be needed for C++ specific optimizations. 979 MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr); 980 break; 981 case LLVMRustOptStage::FatLTO: 982 MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr); 983 break; 984 } 985 } 986 } else { 987 // We're not building any of the default pipelines but we still want to 988 // add the verifier, instrumentation, etc passes if they were requested 989 for (const auto &C : PipelineStartEPCallbacks) 990 C(MPM, OptLevel); 991 for (const auto &C : OptimizerLastEPCallbacks) 992 C(MPM, OptLevel); 993 } 994 995 if (ExtraPassesLen) { 996 if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) { 997 std::string ErrMsg = toString(std::move(Err)); 998 LLVMRustSetLastError(ErrMsg.c_str()); 999 return LLVMRustResult::Failure; 1000 } 1001 } 1002 1003 if (NeedThinLTOBufferPasses) { 1004 MPM.addPass(CanonicalizeAliasesPass()); 1005 MPM.addPass(NameAnonGlobalPass()); 1006 } 1007 1008 // Upgrade all calls to old intrinsics first. 1009 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;) 1010 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove 1011 1012 MPM.run(*TheModule, MAM); 1013 return LLVMRustResult::Success; 1014} 1015 1016// Callback to demangle function name 1017// Parameters: 1018// * name to be demangled 1019// * name len 1020// * output buffer 1021// * output buffer len 1022// Returns len of demangled string, or 0 if demangle failed. 1023typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t); 1024 1025 1026namespace { 1027 1028class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter { 1029 DemangleFn Demangle; 1030 std::vector<char> Buf; 1031 1032public: 1033 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {} 1034 1035 // Return empty string if demangle failed 1036 // or if name does not need to be demangled 1037 StringRef CallDemangle(StringRef name) { 1038 if (!Demangle) { 1039 return StringRef(); 1040 } 1041 1042 if (Buf.size() < name.size() * 2) { 1043 // Semangled name usually shorter than mangled, 1044 // but allocate twice as much memory just in case 1045 Buf.resize(name.size() * 2); 1046 } 1047 1048 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size()); 1049 if (!R) { 1050 // Demangle failed. 1051 return StringRef(); 1052 } 1053 1054 auto Demangled = StringRef(Buf.data(), R); 1055 if (Demangled == name) { 1056 // Do not print anything if demangled name is equal to mangled. 1057 return StringRef(); 1058 } 1059 1060 return Demangled; 1061 } 1062 1063 void emitFunctionAnnot(const Function *F, 1064 formatted_raw_ostream &OS) override { 1065 StringRef Demangled = CallDemangle(F->getName()); 1066 if (Demangled.empty()) { 1067 return; 1068 } 1069 1070 OS << "; " << Demangled << "\n"; 1071 } 1072 1073 void emitInstructionAnnot(const Instruction *I, 1074 formatted_raw_ostream &OS) override { 1075 const char *Name; 1076 const Value *Value; 1077 if (const CallInst *CI = dyn_cast<CallInst>(I)) { 1078 Name = "call"; 1079 Value = CI->getCalledOperand(); 1080 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) { 1081 Name = "invoke"; 1082 Value = II->getCalledOperand(); 1083 } else { 1084 // Could demangle more operations, e. g. 1085 // `store %place, @function`. 1086 return; 1087 } 1088 1089 if (!Value->hasName()) { 1090 return; 1091 } 1092 1093 StringRef Demangled = CallDemangle(Value->getName()); 1094 if (Demangled.empty()) { 1095 return; 1096 } 1097 1098 OS << "; " << Name << " " << Demangled << "\n"; 1099 } 1100}; 1101 1102} // namespace 1103 1104extern "C" LLVMRustResult 1105LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) { 1106 std::string ErrorInfo; 1107 std::error_code EC; 1108 raw_fd_ostream OS(Path, EC, sys::fs::OF_None); 1109 if (EC) 1110 ErrorInfo = EC.message(); 1111 if (ErrorInfo != "") { 1112 LLVMRustSetLastError(ErrorInfo.c_str()); 1113 return LLVMRustResult::Failure; 1114 } 1115 1116 RustAssemblyAnnotationWriter AAW(Demangle); 1117 formatted_raw_ostream FOS(OS); 1118 unwrap(M)->print(FOS, &AAW); 1119 1120 return LLVMRustResult::Success; 1121} 1122 1123extern "C" void LLVMRustPrintPasses() { 1124 PassBuilder PB; 1125 PB.printPassNames(outs()); 1126} 1127 1128extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, 1129 size_t Len) { 1130 auto PreserveFunctions = [=](const GlobalValue &GV) { 1131 // Preserve LLVM-injected, ASAN-related symbols. 1132 // See also https://github.com/rust-lang/rust/issues/113404. 1133 if (GV.getName() == "___asan_globals_registered") { 1134 return true; 1135 } 1136 1137 // Preserve symbols exported from Rust modules. 1138 for (size_t I = 0; I < Len; I++) { 1139 if (GV.getName() == Symbols[I]) { 1140 return true; 1141 } 1142 } 1143 return false; 1144 }; 1145 1146 internalizeModule(*unwrap(M), PreserveFunctions); 1147} 1148 1149extern "C" void 1150LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module, 1151 LLVMTargetMachineRef TMR) { 1152 TargetMachine *Target = unwrap(TMR); 1153 unwrap(Module)->setDataLayout(Target->createDataLayout()); 1154} 1155 1156extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) { 1157 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC); 1158} 1159 1160extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { 1161 unwrap(M)->setPIELevel(PIELevel::Level::Large); 1162} 1163 1164extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M, 1165 LLVMRustCodeModel Model) { 1166 auto CM = fromRust(Model); 1167 if (!CM) 1168 return; 1169 unwrap(M)->setCodeModel(*CM); 1170} 1171 1172// Here you'll find an implementation of ThinLTO as used by the Rust compiler 1173// right now. This ThinLTO support is only enabled on "recent ish" versions of 1174// LLVM, and otherwise it's just blanket rejected from other compilers. 1175// 1176// Most of this implementation is straight copied from LLVM. At the time of 1177// this writing it wasn't *quite* suitable to reuse more code from upstream 1178// for our purposes, but we should strive to upstream this support once it's 1179// ready to go! I figure we may want a bit of testing locally first before 1180// sending this upstream to LLVM. I hear though they're quite eager to receive 1181// feedback like this! 1182// 1183// If you're reading this code and wondering "what in the world" or you're 1184// working "good lord by LLVM upgrade is *still* failing due to these bindings" 1185// then fear not! (ok maybe fear a little). All code here is mostly based 1186// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM. 1187// 1188// You'll find that the general layout here roughly corresponds to the `run` 1189// method in that file as well as `ProcessThinLTOModule`. Functions are 1190// specifically commented below as well, but if you're updating this code 1191// or otherwise trying to understand it, the LLVM source will be useful in 1192// interpreting the mysteries within. 1193// 1194// Otherwise I'll apologize in advance, it probably requires a relatively 1195// significant investment on your part to "truly understand" what's going on 1196// here. Not saying I do myself, but it took me awhile staring at LLVM's source 1197// and various online resources about ThinLTO to make heads or tails of all 1198// this. 1199 1200// This is a shared data structure which *must* be threadsafe to share 1201// read-only amongst threads. This also corresponds basically to the arguments 1202// of the `ProcessThinLTOModule` function in the LLVM source. 1203struct LLVMRustThinLTOData { 1204 // The combined index that is the global analysis over all modules we're 1205 // performing ThinLTO for. This is mostly managed by LLVM. 1206 ModuleSummaryIndex Index; 1207 1208 // All modules we may look at, stored as in-memory serialized versions. This 1209 // is later used when inlining to ensure we can extract any module to inline 1210 // from. 1211 StringMap<MemoryBufferRef> ModuleMap; 1212 1213 // A set that we manage of everything we *don't* want internalized. Note that 1214 // this includes all transitive references right now as well, but it may not 1215 // always! 1216 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols; 1217 1218 // Not 100% sure what these are, but they impact what's internalized and 1219 // what's inlined across modules, I believe. 1220#if LLVM_VERSION_GE(18, 0) 1221 DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists; 1222 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists; 1223 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries; 1224#else 1225 StringMap<FunctionImporter::ImportMapTy> ImportLists; 1226 StringMap<FunctionImporter::ExportSetTy> ExportLists; 1227 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries; 1228#endif 1229 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; 1230 1231 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} 1232}; 1233 1234// Just an argument to the `LLVMRustCreateThinLTOData` function below. 1235struct LLVMRustThinLTOModule { 1236 const char *identifier; 1237 const char *data; 1238 size_t len; 1239}; 1240 1241// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it 1242// does. 1243static const GlobalValueSummary * 1244getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { 1245 auto StrongDefForLinker = llvm::find_if( 1246 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) { 1247 auto Linkage = Summary->linkage(); 1248 return !GlobalValue::isAvailableExternallyLinkage(Linkage) && 1249 !GlobalValue::isWeakForLinker(Linkage); 1250 }); 1251 if (StrongDefForLinker != GVSummaryList.end()) 1252 return StrongDefForLinker->get(); 1253 1254 auto FirstDefForLinker = llvm::find_if( 1255 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) { 1256 auto Linkage = Summary->linkage(); 1257 return !GlobalValue::isAvailableExternallyLinkage(Linkage); 1258 }); 1259 if (FirstDefForLinker == GVSummaryList.end()) 1260 return nullptr; 1261 return FirstDefForLinker->get(); 1262} 1263 1264// The main entry point for creating the global ThinLTO analysis. The structure 1265// here is basically the same as before threads are spawned in the `run` 1266// function of `lib/LTO/ThinLTOCodeGenerator.cpp`. 1267extern "C" LLVMRustThinLTOData* 1268LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, 1269 int num_modules, 1270 const char **preserved_symbols, 1271 int num_symbols) { 1272 auto Ret = std::make_unique<LLVMRustThinLTOData>(); 1273 1274 // Load each module's summary and merge it into one combined index 1275 for (int i = 0; i < num_modules; i++) { 1276 auto module = &modules[i]; 1277 StringRef buffer(module->data, module->len); 1278 MemoryBufferRef mem_buffer(buffer, module->identifier); 1279 1280 Ret->ModuleMap[module->identifier] = mem_buffer; 1281 1282#if LLVM_VERSION_GE(18, 0) 1283 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index)) { 1284#else 1285 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { 1286#endif 1287 LLVMRustSetLastError(toString(std::move(Err)).c_str()); 1288 return nullptr; 1289 } 1290 } 1291 1292 // Collect for each module the list of function it defines (GUID -> Summary) 1293 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries); 1294 1295 // Convert the preserved symbols set from string to GUID, this is then needed 1296 // for internalization. 1297 for (int i = 0; i < num_symbols; i++) { 1298 auto GUID = GlobalValue::getGUID(preserved_symbols[i]); 1299 Ret->GUIDPreservedSymbols.insert(GUID); 1300 } 1301 1302 // Collect the import/export lists for all modules from the call-graph in the 1303 // combined index 1304 // 1305 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` 1306 auto deadIsPrevailing = [&](GlobalValue::GUID G) { 1307 return PrevailingType::Unknown; 1308 }; 1309 // We don't have a complete picture in our use of ThinLTO, just our immediate 1310 // crate, so we need `ImportEnabled = false` to limit internalization. 1311 // Otherwise, we sometimes lose `static` values -- see #60184. 1312 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, 1313 deadIsPrevailing, /* ImportEnabled = */ false); 1314 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it 1315 // impacts the caching. 1316 // 1317 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this 1318 // being lifted from `lib/LTO/LTO.cpp` as well 1319 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; 1320 for (auto &I : Ret->Index) { 1321 if (I.second.SummaryList.size() > 1) 1322 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList); 1323 } 1324 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { 1325 const auto &Prevailing = PrevailingCopy.find(GUID); 1326 if (Prevailing == PrevailingCopy.end()) 1327 return true; 1328 return Prevailing->second == S; 1329 }; 1330 ComputeCrossModuleImport( 1331 Ret->Index, 1332 Ret->ModuleToDefinedGVSummaries, 1333#if LLVM_VERSION_GE(17, 0) 1334 isPrevailing, 1335#endif 1336 Ret->ImportLists, 1337 Ret->ExportLists 1338 ); 1339 1340 auto recordNewLinkage = [&](StringRef ModuleIdentifier, 1341 GlobalValue::GUID GUID, 1342 GlobalValue::LinkageTypes NewLinkage) { 1343 Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; 1344 }; 1345 1346 // Uses FromPrevailing visibility scheme which works for many binary 1347 // formats. We probably could and should use ELF visibility scheme for many of 1348 // our targets, however. 1349 lto::Config conf; 1350 thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage, 1351 Ret->GUIDPreservedSymbols); 1352 1353 // Here we calculate an `ExportedGUIDs` set for use in the `isExported` 1354 // callback below. This callback below will dictate the linkage for all 1355 // summaries in the index, and we basically just only want to ensure that dead 1356 // symbols are internalized. Otherwise everything that's already external 1357 // linkage will stay as external, and internal will stay as internal. 1358 std::set<GlobalValue::GUID> ExportedGUIDs; 1359 for (auto &List : Ret->Index) { 1360 for (auto &GVS: List.second.SummaryList) { 1361 if (GlobalValue::isLocalLinkage(GVS->linkage())) 1362 continue; 1363 auto GUID = GVS->getOriginalName(); 1364 if (GVS->flags().Live) 1365 ExportedGUIDs.insert(GUID); 1366 } 1367 } 1368 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) { 1369 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); 1370 return (ExportList != Ret->ExportLists.end() && 1371 ExportList->second.count(VI)) || 1372 ExportedGUIDs.count(VI.getGUID()); 1373 }; 1374 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing); 1375 1376 return Ret.release(); 1377} 1378 1379extern "C" void 1380LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { 1381 delete Data; 1382} 1383 1384// Below are the various passes that happen *per module* when doing ThinLTO. 1385// 1386// In other words, these are the functions that are all run concurrently 1387// with one another, one per module. The passes here correspond to the analysis 1388// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the 1389// `ProcessThinLTOModule` function. Here they're split up into separate steps 1390// so rustc can save off the intermediate bytecode between each step. 1391 1392static bool 1393clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) { 1394 // When linking an ELF shared object, dso_local should be dropped. We 1395 // conservatively do this for -fpic. 1396 bool ClearDSOLocalOnDeclarations = 1397 TM.getTargetTriple().isOSBinFormatELF() && 1398 TM.getRelocationModel() != Reloc::Static && 1399 Mod.getPIELevel() == PIELevel::Default; 1400 return ClearDSOLocalOnDeclarations; 1401} 1402 1403extern "C" bool 1404LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M, 1405 LLVMTargetMachineRef TM) { 1406 Module &Mod = *unwrap(M); 1407 TargetMachine &Target = *unwrap(TM); 1408 1409 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); 1410 bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal); 1411 1412 if (error) { 1413 LLVMRustSetLastError("renameModuleForThinLTO failed"); 1414 return false; 1415 } 1416 return true; 1417} 1418 1419extern "C" bool 1420LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 1421 Module &Mod = *unwrap(M); 1422 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); 1423 thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true); 1424 return true; 1425} 1426 1427extern "C" bool 1428LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 1429 Module &Mod = *unwrap(M); 1430 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); 1431 thinLTOInternalizeModule(Mod, DefinedGlobals); 1432 return true; 1433} 1434 1435extern "C" bool 1436LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M, 1437 LLVMTargetMachineRef TM) { 1438 Module &Mod = *unwrap(M); 1439 TargetMachine &Target = *unwrap(TM); 1440 1441 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier()); 1442 auto Loader = [&](StringRef Identifier) { 1443 const auto &Memory = Data->ModuleMap.lookup(Identifier); 1444 auto &Context = Mod.getContext(); 1445 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true); 1446 1447 if (!MOrErr) 1448 return MOrErr; 1449 1450 // The rest of this closure is a workaround for 1451 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports 1452 // we accidentally import wasm custom sections into different modules, 1453 // duplicating them by in the final output artifact. 1454 // 1455 // The issue is worked around here by manually removing the 1456 // `wasm.custom_sections` named metadata node from any imported module. This 1457 // we know isn't used by any optimization pass so there's no need for it to 1458 // be imported. 1459 // 1460 // Note that the metadata is currently lazily loaded, so we materialize it 1461 // here before looking up if there's metadata inside. The `FunctionImporter` 1462 // will immediately materialize metadata anyway after an import, so this 1463 // shouldn't be a perf hit. 1464 if (Error Err = (*MOrErr)->materializeMetadata()) { 1465 Expected<std::unique_ptr<Module>> Ret(std::move(Err)); 1466 return Ret; 1467 } 1468 1469 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections"); 1470 if (WasmCustomSections) 1471 WasmCustomSections->eraseFromParent(); 1472 1473 // `llvm.ident` named metadata also gets duplicated. 1474 auto *llvmIdent = (*MOrErr)->getNamedMetadata("llvm.ident"); 1475 if (llvmIdent) 1476 llvmIdent->eraseFromParent(); 1477 1478 return MOrErr; 1479 }; 1480 bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); 1481 FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal); 1482 Expected<bool> Result = Importer.importFunctions(Mod, ImportList); 1483 if (!Result) { 1484 LLVMRustSetLastError(toString(Result.takeError()).c_str()); 1485 return false; 1486 } 1487 return true; 1488} 1489 1490// This struct and various functions are sort of a hack right now, but the 1491// problem is that we've got in-memory LLVM modules after we generate and 1492// optimize all codegen-units for one compilation in rustc. To be compatible 1493// with the LTO support above we need to serialize the modules plus their 1494// ThinLTO summary into memory. 1495// 1496// This structure is basically an owned version of a serialize module, with 1497// a ThinLTO summary attached. 1498struct LLVMRustThinLTOBuffer { 1499 std::string data; 1500}; 1501 1502extern "C" LLVMRustThinLTOBuffer* 1503LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) { 1504 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>(); 1505 { 1506 raw_string_ostream OS(Ret->data); 1507 { 1508 if (is_thin) { 1509 PassBuilder PB; 1510 LoopAnalysisManager LAM; 1511 FunctionAnalysisManager FAM; 1512 CGSCCAnalysisManager CGAM; 1513 ModuleAnalysisManager MAM; 1514 PB.registerModuleAnalyses(MAM); 1515 PB.registerCGSCCAnalyses(CGAM); 1516 PB.registerFunctionAnalyses(FAM); 1517 PB.registerLoopAnalyses(LAM); 1518 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); 1519 ModulePassManager MPM; 1520 MPM.addPass(ThinLTOBitcodeWriterPass(OS, nullptr)); 1521 MPM.run(*unwrap(M), MAM); 1522 } else { 1523 WriteBitcodeToFile(*unwrap(M), OS); 1524 } 1525 } 1526 } 1527 return Ret.release(); 1528} 1529 1530extern "C" void 1531LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { 1532 delete Buffer; 1533} 1534 1535extern "C" const void* 1536LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { 1537 return Buffer->data.data(); 1538} 1539 1540extern "C" size_t 1541LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { 1542 return Buffer->data.length(); 1543} 1544 1545// This is what we used to parse upstream bitcode for actual ThinLTO 1546// processing. We'll call this once per module optimized through ThinLTO, and 1547// it'll be called concurrently on many threads. 1548extern "C" LLVMModuleRef 1549LLVMRustParseBitcodeForLTO(LLVMContextRef Context, 1550 const char *data, 1551 size_t len, 1552 const char *identifier) { 1553 StringRef Data(data, len); 1554 MemoryBufferRef Buffer(Data, identifier); 1555 unwrap(Context)->enableDebugTypeODRUniquing(); 1556 Expected<std::unique_ptr<Module>> SrcOrError = 1557 parseBitcodeFile(Buffer, *unwrap(Context)); 1558 if (!SrcOrError) { 1559 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); 1560 return nullptr; 1561 } 1562 return wrap(std::move(*SrcOrError).release()); 1563} 1564 1565// Find the bitcode section in the object file data and return it as a slice. 1566// Fail if the bitcode section is present but empty. 1567// 1568// On success, the return value is the pointer to the start of the slice and 1569// `out_len` is filled with the (non-zero) length. On failure, the return value 1570// is `nullptr` and `out_len` is set to zero. 1571extern "C" const char* 1572LLVMRustGetBitcodeSliceFromObjectData(const char *data, 1573 size_t len, 1574 size_t *out_len) { 1575 *out_len = 0; 1576 1577 StringRef Data(data, len); 1578 MemoryBufferRef Buffer(Data, ""); // The id is unused. 1579 1580 Expected<MemoryBufferRef> BitcodeOrError = 1581 object::IRObjectFile::findBitcodeInMemBuffer(Buffer); 1582 if (!BitcodeOrError) { 1583 LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str()); 1584 return nullptr; 1585 } 1586 1587 *out_len = BitcodeOrError->getBufferSize(); 1588 return BitcodeOrError->getBufferStart(); 1589} 1590 1591// Find a section of an object file by name. Fail if the section is missing or 1592// empty. 1593extern "C" const char *LLVMRustGetSliceFromObjectDataByName(const char *data, 1594 size_t len, 1595 const char *name, 1596 size_t *out_len) { 1597 *out_len = 0; 1598 StringRef Data(data, len); 1599 MemoryBufferRef Buffer(Data, ""); // The id is unused. 1600 file_magic Type = identify_magic(Buffer.getBuffer()); 1601 Expected<std::unique_ptr<object::ObjectFile>> ObjFileOrError = 1602 object::ObjectFile::createObjectFile(Buffer, Type); 1603 if (!ObjFileOrError) { 1604 LLVMRustSetLastError(toString(ObjFileOrError.takeError()).c_str()); 1605 return nullptr; 1606 } 1607 for (const object::SectionRef &Sec : (*ObjFileOrError)->sections()) { 1608 Expected<StringRef> Name = Sec.getName(); 1609 if (Name && *Name == name) { 1610 Expected<StringRef> SectionOrError = Sec.getContents(); 1611 if (!SectionOrError) { 1612 LLVMRustSetLastError(toString(SectionOrError.takeError()).c_str()); 1613 return nullptr; 1614 } 1615 *out_len = SectionOrError->size(); 1616 return SectionOrError->data(); 1617 } 1618 } 1619 LLVMRustSetLastError("could not find requested section"); 1620 return nullptr; 1621} 1622 1623// Computes the LTO cache key for the provided 'ModId' in the given 'Data', 1624// storing the result in 'KeyOut'. 1625// Currently, this cache key is a SHA-1 hash of anything that could affect 1626// the result of optimizing this module (e.g. module imports, exports, liveness 1627// of access globals, etc). 1628// The precise details are determined by LLVM in `computeLTOCacheKey`, which is 1629// used during the normal linker-plugin incremental thin-LTO process. 1630extern "C" void 1631LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) { 1632 SmallString<40> Key; 1633 llvm::lto::Config conf; 1634 const auto &ImportList = Data->ImportLists.lookup(ModId); 1635 const auto &ExportList = Data->ExportLists.lookup(ModId); 1636 const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId); 1637 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId); 1638 std::set<GlobalValue::GUID> CfiFunctionDefs; 1639 std::set<GlobalValue::GUID> CfiFunctionDecls; 1640 1641 // Based on the 'InProcessThinBackend' constructor in LLVM 1642 for (auto &Name : Data->Index.cfiFunctionDefs()) 1643 CfiFunctionDefs.insert( 1644 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); 1645 for (auto &Name : Data->Index.cfiFunctionDecls()) 1646 CfiFunctionDecls.insert( 1647 GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); 1648 1649 llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId, 1650 ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls 1651 ); 1652 1653 LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size()); 1654}