fork/rust
git clone https://git.julia.blue/fork/rust
49cae55760d
1.44.0
1#include <stdio.h> 2 3#include <vector> 4#include <set> 5 6#include "rustllvm.h" 7 8#include "llvm/Analysis/TargetLibraryInfo.h" 9#include "llvm/Analysis/TargetTransformInfo.h" 10#include "llvm/CodeGen/TargetSubtargetInfo.h" 11#include "llvm/InitializePasses.h" 12#include "llvm/IR/AutoUpgrade.h" 13#include "llvm/IR/AssemblyAnnotationWriter.h" 14#include "llvm/IR/IntrinsicInst.h" 15#include "llvm/IR/Verifier.h" 16#include "llvm/Passes/PassBuilder.h" 17#if LLVM_VERSION_GE(9, 0) 18#include "llvm/Passes/StandardInstrumentations.h" 19#endif 20#include "llvm/Support/CBindingWrapping.h" 21#include "llvm/Support/FileSystem.h" 22#include "llvm/Support/Host.h" 23#include "llvm/Target/TargetMachine.h" 24#include "llvm/Transforms/IPO/PassManagerBuilder.h" 25#include "llvm/Transforms/IPO/AlwaysInliner.h" 26#include "llvm/Transforms/IPO/FunctionImport.h" 27#include "llvm/Transforms/Utils/FunctionImportUtils.h" 28#include "llvm/LTO/LTO.h" 29#include "llvm-c/Transforms/PassManagerBuilder.h" 30 31#include "llvm/Transforms/Instrumentation.h" 32#if LLVM_VERSION_GE(9, 0) 33#include "llvm/Transforms/Instrumentation/AddressSanitizer.h" 34#include "llvm/Support/TimeProfiler.h" 35#endif 36#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" 37#include "llvm/Transforms/Instrumentation/MemorySanitizer.h" 38#if LLVM_VERSION_GE(9, 0) 39#include "llvm/Transforms/Utils/CanonicalizeAliases.h" 40#endif 41#include "llvm/Transforms/Utils/NameAnonGlobals.h" 42 43using namespace llvm; 44 45typedef struct LLVMOpaquePass *LLVMPassRef; 46typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; 47 48DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef) 49DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) 50DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder, 51 LLVMPassManagerBuilderRef) 52 53extern "C" void LLVMInitializePasses() { 54 PassRegistry &Registry = *PassRegistry::getPassRegistry(); 55 initializeCore(Registry); 56 initializeCodeGen(Registry); 57 initializeScalarOpts(Registry); 58 initializeVectorization(Registry); 59 initializeIPO(Registry); 60 initializeAnalysis(Registry); 61 initializeTransformUtils(Registry); 62 initializeInstCombine(Registry); 63 initializeInstrumentation(Registry); 64 initializeTarget(Registry); 65} 66 67extern "C" void LLVMTimeTraceProfilerInitialize() { 68#if LLVM_VERSION_GE(10, 0) 69 timeTraceProfilerInitialize( 70 /* TimeTraceGranularity */ 0, 71 /* ProcName */ "rustc"); 72#elif LLVM_VERSION_GE(9, 0) 73 timeTraceProfilerInitialize(); 74#endif 75} 76 77extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) { 78#if LLVM_VERSION_GE(9, 0) 79 StringRef FN(FileName); 80 std::error_code EC; 81 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways); 82 83 timeTraceProfilerWrite(OS); 84 timeTraceProfilerCleanup(); 85#endif 86} 87 88enum class LLVMRustPassKind { 89 Other, 90 Function, 91 Module, 92}; 93 94static LLVMRustPassKind toRust(PassKind Kind) { 95 switch (Kind) { 96 case PT_Function: 97 return LLVMRustPassKind::Function; 98 case PT_Module: 99 return LLVMRustPassKind::Module; 100 default: 101 return LLVMRustPassKind::Other; 102 } 103} 104 105extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) { 106 StringRef SR(PassName); 107 PassRegistry *PR = PassRegistry::getPassRegistry(); 108 109 const PassInfo *PI = PR->getPassInfo(SR); 110 if (PI) { 111 return wrap(PI->createPass()); 112 } 113 return nullptr; 114} 115 116extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) { 117 const bool CompileKernel = false; 118 const bool UseAfterScope = true; 119 120 return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope)); 121} 122 123extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) { 124 const bool CompileKernel = false; 125 126#if LLVM_VERSION_GE(9, 0) 127 return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover)); 128#else 129 return wrap(createAddressSanitizerModulePass(CompileKernel, Recover)); 130#endif 131} 132 133extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) { 134#if LLVM_VERSION_GE(9, 0) 135 const bool CompileKernel = false; 136 137 return wrap(createMemorySanitizerLegacyPassPass( 138 MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel})); 139#else 140 return wrap(createMemorySanitizerLegacyPassPass(TrackOrigins, Recover)); 141#endif 142} 143 144extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() { 145 return wrap(createThreadSanitizerLegacyPassPass()); 146} 147 148extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) { 149 assert(RustPass); 150 Pass *Pass = unwrap(RustPass); 151 return toRust(Pass->getPassKind()); 152} 153 154extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) { 155 assert(RustPass); 156 Pass *Pass = unwrap(RustPass); 157 PassManagerBase *PMB = unwrap(PMR); 158 PMB->add(Pass); 159} 160 161extern "C" 162void LLVMRustPassManagerBuilderPopulateThinLTOPassManager( 163 LLVMPassManagerBuilderRef PMBR, 164 LLVMPassManagerRef PMR 165) { 166 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR)); 167} 168 169extern "C" 170void LLVMRustAddLastExtensionPasses( 171 LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) { 172 auto AddExtensionPasses = [Passes, NumPasses]( 173 const PassManagerBuilder &Builder, PassManagerBase &PM) { 174 for (size_t I = 0; I < NumPasses; I++) { 175 PM.add(unwrap(Passes[I])); 176 } 177 }; 178 // Add the passes to both of the pre-finalization extension points, 179 // so they are run for optimized and non-optimized builds. 180 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast, 181 AddExtensionPasses); 182 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, 183 AddExtensionPasses); 184} 185 186#ifdef LLVM_COMPONENT_X86 187#define SUBTARGET_X86 SUBTARGET(X86) 188#else 189#define SUBTARGET_X86 190#endif 191 192#ifdef LLVM_COMPONENT_ARM 193#define SUBTARGET_ARM SUBTARGET(ARM) 194#else 195#define SUBTARGET_ARM 196#endif 197 198#ifdef LLVM_COMPONENT_AARCH64 199#define SUBTARGET_AARCH64 SUBTARGET(AArch64) 200#else 201#define SUBTARGET_AARCH64 202#endif 203 204#ifdef LLVM_COMPONENT_MIPS 205#define SUBTARGET_MIPS SUBTARGET(Mips) 206#else 207#define SUBTARGET_MIPS 208#endif 209 210#ifdef LLVM_COMPONENT_POWERPC 211#define SUBTARGET_PPC SUBTARGET(PPC) 212#else 213#define SUBTARGET_PPC 214#endif 215 216#ifdef LLVM_COMPONENT_SYSTEMZ 217#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ) 218#else 219#define SUBTARGET_SYSTEMZ 220#endif 221 222#ifdef LLVM_COMPONENT_MSP430 223#define SUBTARGET_MSP430 SUBTARGET(MSP430) 224#else 225#define SUBTARGET_MSP430 226#endif 227 228#ifdef LLVM_COMPONENT_RISCV 229#define SUBTARGET_RISCV SUBTARGET(RISCV) 230#else 231#define SUBTARGET_RISCV 232#endif 233 234#ifdef LLVM_COMPONENT_SPARC 235#define SUBTARGET_SPARC SUBTARGET(Sparc) 236#else 237#define SUBTARGET_SPARC 238#endif 239 240#ifdef LLVM_COMPONENT_HEXAGON 241#define SUBTARGET_HEXAGON SUBTARGET(Hexagon) 242#else 243#define SUBTARGET_HEXAGON 244#endif 245 246#define GEN_SUBTARGETS \ 247 SUBTARGET_X86 \ 248 SUBTARGET_ARM \ 249 SUBTARGET_AARCH64 \ 250 SUBTARGET_MIPS \ 251 SUBTARGET_PPC \ 252 SUBTARGET_SYSTEMZ \ 253 SUBTARGET_MSP430 \ 254 SUBTARGET_SPARC \ 255 SUBTARGET_HEXAGON \ 256 SUBTARGET_RISCV \ 257 258#define SUBTARGET(x) \ 259 namespace llvm { \ 260 extern const SubtargetFeatureKV x##FeatureKV[]; \ 261 extern const SubtargetFeatureKV x##SubTypeKV[]; \ 262 } 263 264GEN_SUBTARGETS 265#undef SUBTARGET 266 267extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, 268 const char *Feature) { 269 TargetMachine *Target = unwrap(TM); 270 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 271 return MCInfo->checkFeatures(std::string("+") + Feature); 272} 273 274enum class LLVMRustCodeModel { 275 Other, 276 Small, 277 Kernel, 278 Medium, 279 Large, 280 None, 281}; 282 283static CodeModel::Model fromRust(LLVMRustCodeModel Model) { 284 switch (Model) { 285 case LLVMRustCodeModel::Small: 286 return CodeModel::Small; 287 case LLVMRustCodeModel::Kernel: 288 return CodeModel::Kernel; 289 case LLVMRustCodeModel::Medium: 290 return CodeModel::Medium; 291 case LLVMRustCodeModel::Large: 292 return CodeModel::Large; 293 default: 294 report_fatal_error("Bad CodeModel."); 295 } 296} 297 298enum class LLVMRustCodeGenOptLevel { 299 Other, 300 None, 301 Less, 302 Default, 303 Aggressive, 304}; 305 306static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) { 307 switch (Level) { 308 case LLVMRustCodeGenOptLevel::None: 309 return CodeGenOpt::None; 310 case LLVMRustCodeGenOptLevel::Less: 311 return CodeGenOpt::Less; 312 case LLVMRustCodeGenOptLevel::Default: 313 return CodeGenOpt::Default; 314 case LLVMRustCodeGenOptLevel::Aggressive: 315 return CodeGenOpt::Aggressive; 316 default: 317 report_fatal_error("Bad CodeGenOptLevel."); 318 } 319} 320 321enum class LLVMRustPassBuilderOptLevel { 322 O0, 323 O1, 324 O2, 325 O3, 326 Os, 327 Oz, 328}; 329 330static PassBuilder::OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) { 331 switch (Level) { 332 case LLVMRustPassBuilderOptLevel::O0: 333 return PassBuilder::O0; 334 case LLVMRustPassBuilderOptLevel::O1: 335 return PassBuilder::O1; 336 case LLVMRustPassBuilderOptLevel::O2: 337 return PassBuilder::O2; 338 case LLVMRustPassBuilderOptLevel::O3: 339 return PassBuilder::O3; 340 case LLVMRustPassBuilderOptLevel::Os: 341 return PassBuilder::Os; 342 case LLVMRustPassBuilderOptLevel::Oz: 343 return PassBuilder::Oz; 344 default: 345 report_fatal_error("Bad PassBuilderOptLevel."); 346 } 347} 348 349enum class LLVMRustRelocMode { 350 Default, 351 Static, 352 PIC, 353 DynamicNoPic, 354 ROPI, 355 RWPI, 356 ROPIRWPI, 357}; 358 359static Optional<Reloc::Model> fromRust(LLVMRustRelocMode RustReloc) { 360 switch (RustReloc) { 361 case LLVMRustRelocMode::Default: 362 return None; 363 case LLVMRustRelocMode::Static: 364 return Reloc::Static; 365 case LLVMRustRelocMode::PIC: 366 return Reloc::PIC_; 367 case LLVMRustRelocMode::DynamicNoPic: 368 return Reloc::DynamicNoPIC; 369 case LLVMRustRelocMode::ROPI: 370 return Reloc::ROPI; 371 case LLVMRustRelocMode::RWPI: 372 return Reloc::RWPI; 373 case LLVMRustRelocMode::ROPIRWPI: 374 return Reloc::ROPI_RWPI; 375 } 376 report_fatal_error("Bad RelocModel."); 377} 378 379#ifdef LLVM_RUSTLLVM 380/// getLongestEntryLength - Return the length of the longest entry in the table. 381template<typename KV> 382static size_t getLongestEntryLength(ArrayRef<KV> Table) { 383 size_t MaxLen = 0; 384 for (auto &I : Table) 385 MaxLen = std::max(MaxLen, std::strlen(I.Key)); 386 return MaxLen; 387} 388 389extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) { 390 const TargetMachine *Target = unwrap(TM); 391 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 392 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch(); 393 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); 394 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable(); 395 unsigned MaxCPULen = getLongestEntryLength(CPUTable); 396 397 printf("Available CPUs for this target:\n"); 398 if (HostArch == TargetArch) { 399 const StringRef HostCPU = sys::getHostCPUName(); 400 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n", 401 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data()); 402 } 403 for (auto &CPU : CPUTable) 404 printf(" %-*s\n", MaxCPULen, CPU.Key); 405 printf("\n"); 406} 407 408extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) { 409 const TargetMachine *Target = unwrap(TM); 410 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 411 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable(); 412 unsigned MaxFeatLen = getLongestEntryLength(FeatTable); 413 414 printf("Available features for this target:\n"); 415 for (auto &Feature : FeatTable) 416 printf(" %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc); 417 printf("\n"); 418 419 printf("Use +feature to enable a feature, or -feature to disable it.\n" 420 "For example, rustc -C -target-cpu=mycpu -C " 421 "target-feature=+feature1,-feature2\n\n"); 422} 423 424#else 425 426extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) { 427 printf("Target CPU help is not supported by this LLVM version.\n\n"); 428} 429 430extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) { 431 printf("Target features help is not supported by this LLVM version.\n\n"); 432} 433#endif 434 435extern "C" const char* LLVMRustGetHostCPUName(size_t *len) { 436 StringRef Name = sys::getHostCPUName(); 437 *len = Name.size(); 438 return Name.data(); 439} 440 441extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( 442 const char *TripleStr, const char *CPU, const char *Feature, 443 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocMode RustReloc, 444 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, 445 bool PositionIndependentExecutable, bool FunctionSections, 446 bool DataSections, 447 bool TrapUnreachable, 448 bool Singlethread, 449 bool AsmComments, 450 bool EmitStackSizeSection, 451 bool RelaxELFRelocations) { 452 453 auto OptLevel = fromRust(RustOptLevel); 454 auto RM = fromRust(RustReloc); 455 456 std::string Error; 457 Triple Trip(Triple::normalize(TripleStr)); 458 const llvm::Target *TheTarget = 459 TargetRegistry::lookupTarget(Trip.getTriple(), Error); 460 if (TheTarget == nullptr) { 461 LLVMRustSetLastError(Error.c_str()); 462 return nullptr; 463 } 464 465 TargetOptions Options; 466 467 Options.FloatABIType = FloatABI::Default; 468 if (UseSoftFloat) { 469 Options.FloatABIType = FloatABI::Soft; 470 } 471 Options.DataSections = DataSections; 472 Options.FunctionSections = FunctionSections; 473 Options.MCOptions.AsmVerbose = AsmComments; 474 Options.MCOptions.PreserveAsmComments = AsmComments; 475 Options.MCOptions.ABIName = ABIStr; 476 Options.RelaxELFRelocations = RelaxELFRelocations; 477 478 if (TrapUnreachable) { 479 // Tell LLVM to codegen `unreachable` into an explicit trap instruction. 480 // This limits the extent of possible undefined behavior in some cases, as 481 // it prevents control flow from "falling through" into whatever code 482 // happens to be laid out next in memory. 483 Options.TrapUnreachable = true; 484 } 485 486 if (Singlethread) { 487 Options.ThreadModel = ThreadModel::Single; 488 } 489 490 Options.EmitStackSizeSection = EmitStackSizeSection; 491 492 Optional<CodeModel::Model> CM; 493 if (RustCM != LLVMRustCodeModel::None) 494 CM = fromRust(RustCM); 495 TargetMachine *TM = TheTarget->createTargetMachine( 496 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); 497 return wrap(TM); 498} 499 500extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { 501 delete unwrap(TM); 502} 503 504extern "C" void LLVMRustConfigurePassManagerBuilder( 505 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel, 506 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO, 507 const char* PGOGenPath, const char* PGOUsePath) { 508 unwrap(PMBR)->MergeFunctions = MergeFunctions; 509 unwrap(PMBR)->SLPVectorize = SLPVectorize; 510 unwrap(PMBR)->OptLevel = fromRust(OptLevel); 511 unwrap(PMBR)->LoopVectorize = LoopVectorize; 512 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO; 513 514 if (PGOGenPath) { 515 assert(!PGOUsePath); 516 unwrap(PMBR)->EnablePGOInstrGen = true; 517 unwrap(PMBR)->PGOInstrGen = PGOGenPath; 518 } 519 if (PGOUsePath) { 520 assert(!PGOGenPath); 521 unwrap(PMBR)->PGOInstrUse = PGOUsePath; 522 } 523} 524 525// Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo` 526// field of a PassManagerBuilder, we expose our own method of doing so. 527extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR, 528 LLVMModuleRef M, 529 bool DisableSimplifyLibCalls) { 530 Triple TargetTriple(unwrap(M)->getTargetTriple()); 531 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple); 532 if (DisableSimplifyLibCalls) 533 TLI->disableAllFunctions(); 534 unwrap(PMBR)->LibraryInfo = TLI; 535} 536 537// Unfortunately, the LLVM C API doesn't provide a way to create the 538// TargetLibraryInfo pass, so we use this method to do so. 539extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M, 540 bool DisableSimplifyLibCalls) { 541 Triple TargetTriple(unwrap(M)->getTargetTriple()); 542 TargetLibraryInfoImpl TLII(TargetTriple); 543 if (DisableSimplifyLibCalls) 544 TLII.disableAllFunctions(); 545 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII)); 546} 547 548// Unfortunately, the LLVM C API doesn't provide an easy way of iterating over 549// all the functions in a module, so we do that manually here. You'll find 550// similar code in clang's BackendUtil.cpp file. 551extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR, 552 LLVMModuleRef M) { 553 llvm::legacy::FunctionPassManager *P = 554 unwrap<llvm::legacy::FunctionPassManager>(PMR); 555 P->doInitialization(); 556 557 // Upgrade all calls to old intrinsics first. 558 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;) 559 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove 560 561 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E; 562 ++I) 563 if (!I->isDeclaration()) 564 P->run(*I); 565 566 P->doFinalization(); 567} 568 569extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) { 570 // Initializing the command-line options more than once is not allowed. So, 571 // check if they've already been initialized. (This could happen if we're 572 // being called from rustpkg, for example). If the arguments change, then 573 // that's just kinda unfortunate. 574 static bool Initialized = false; 575 if (Initialized) 576 return; 577 Initialized = true; 578 cl::ParseCommandLineOptions(Argc, Argv); 579} 580 581enum class LLVMRustFileType { 582 Other, 583 AssemblyFile, 584 ObjectFile, 585}; 586 587#if LLVM_VERSION_GE(10, 0) 588static CodeGenFileType fromRust(LLVMRustFileType Type) { 589 switch (Type) { 590 case LLVMRustFileType::AssemblyFile: 591 return CGFT_AssemblyFile; 592 case LLVMRustFileType::ObjectFile: 593 return CGFT_ObjectFile; 594 default: 595 report_fatal_error("Bad FileType."); 596 } 597} 598#else 599static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) { 600 switch (Type) { 601 case LLVMRustFileType::AssemblyFile: 602 return TargetMachine::CGFT_AssemblyFile; 603 case LLVMRustFileType::ObjectFile: 604 return TargetMachine::CGFT_ObjectFile; 605 default: 606 report_fatal_error("Bad FileType."); 607 } 608} 609#endif 610 611extern "C" LLVMRustResult 612LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, 613 LLVMModuleRef M, const char *Path, 614 LLVMRustFileType RustFileType) { 615 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR); 616 auto FileType = fromRust(RustFileType); 617 618 std::string ErrorInfo; 619 std::error_code EC; 620 raw_fd_ostream OS(Path, EC, sys::fs::F_None); 621 if (EC) 622 ErrorInfo = EC.message(); 623 if (ErrorInfo != "") { 624 LLVMRustSetLastError(ErrorInfo.c_str()); 625 return LLVMRustResult::Failure; 626 } 627 628 buffer_ostream BOS(OS); 629 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false); 630 PM->run(*unwrap(M)); 631 632 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output 633 // stream (OS), so the only real safe place to delete this is here? Don't we 634 // wish this was written in Rust? 635 LLVMDisposePassManager(PMR); 636 return LLVMRustResult::Success; 637} 638 639extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler 640 const char*, // pass name 641 const char*); // IR name 642extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler 643 644#if LLVM_VERSION_GE(9, 0) 645 646std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { 647 if (any_isa<const Module *>(WrappedIr)) 648 return any_cast<const Module *>(WrappedIr)->getName().str(); 649 if (any_isa<const Function *>(WrappedIr)) 650 return any_cast<const Function *>(WrappedIr)->getName().str(); 651 if (any_isa<const Loop *>(WrappedIr)) 652 return any_cast<const Loop *>(WrappedIr)->getName().str(); 653 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr)) 654 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName(); 655 return "<UNKNOWN>"; 656} 657 658 659void LLVMSelfProfileInitializeCallbacks( 660 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler, 661 LLVMRustSelfProfileBeforePassCallback BeforePassCallback, 662 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { 663 PIC.registerBeforePassCallback([LlvmSelfProfiler, BeforePassCallback]( 664 StringRef Pass, llvm::Any Ir) { 665 std::string PassName = Pass.str(); 666 std::string IrName = LLVMRustwrappedIrGetName(Ir); 667 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); 668 return true; 669 }); 670 671 PIC.registerAfterPassCallback( 672 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { 673 AfterPassCallback(LlvmSelfProfiler); 674 }); 675 676 PIC.registerAfterPassInvalidatedCallback( 677 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass) { 678 AfterPassCallback(LlvmSelfProfiler); 679 }); 680 681 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback]( 682 StringRef Pass, llvm::Any Ir) { 683 std::string PassName = Pass.str(); 684 std::string IrName = LLVMRustwrappedIrGetName(Ir); 685 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); 686 }); 687 688 PIC.registerAfterAnalysisCallback( 689 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { 690 AfterPassCallback(LlvmSelfProfiler); 691 }); 692} 693#endif 694 695enum class LLVMRustOptStage { 696 PreLinkNoLTO, 697 PreLinkThinLTO, 698 PreLinkFatLTO, 699 ThinLTO, 700 FatLTO, 701}; 702 703struct LLVMRustSanitizerOptions { 704 bool SanitizeMemory; 705 bool SanitizeThread; 706 bool SanitizeAddress; 707 bool SanitizeRecover; 708 int SanitizeMemoryTrackOrigins; 709}; 710 711extern "C" void 712LLVMRustOptimizeWithNewPassManager( 713 LLVMModuleRef ModuleRef, 714 LLVMTargetMachineRef TMRef, 715 LLVMRustPassBuilderOptLevel OptLevelRust, 716 LLVMRustOptStage OptStage, 717 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers, 718 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, 719 bool DisableSimplifyLibCalls, 720 LLVMRustSanitizerOptions *SanitizerOptions, 721 const char *PGOGenPath, const char *PGOUsePath, 722 void* LlvmSelfProfiler, 723 LLVMRustSelfProfileBeforePassCallback BeforePassCallback, 724 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { 725#if LLVM_VERSION_GE(9, 0) 726 Module *TheModule = unwrap(ModuleRef); 727 TargetMachine *TM = unwrap(TMRef); 728 PassBuilder::OptimizationLevel OptLevel = fromRust(OptLevelRust); 729 730 // FIXME: MergeFunctions is not supported by NewPM yet. 731 (void) MergeFunctions; 732 733 PipelineTuningOptions PTO; 734 PTO.LoopUnrolling = UnrollLoops; 735 PTO.LoopInterleaving = UnrollLoops; 736 PTO.LoopVectorization = LoopVectorize; 737 PTO.SLPVectorization = SLPVectorize; 738 739 PassInstrumentationCallbacks PIC; 740 StandardInstrumentations SI; 741 SI.registerCallbacks(PIC); 742 743 if (LlvmSelfProfiler){ 744 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback); 745 } 746 747 Optional<PGOOptions> PGOOpt; 748 if (PGOGenPath) { 749 assert(!PGOUsePath); 750 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr); 751 } else if (PGOUsePath) { 752 assert(!PGOGenPath); 753 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse); 754 } 755 756 PassBuilder PB(TM, PTO, PGOOpt, &PIC); 757 758 // FIXME: We may want to expose this as an option. 759 bool DebugPassManager = false; 760 LoopAnalysisManager LAM(DebugPassManager); 761 FunctionAnalysisManager FAM(DebugPassManager); 762 CGSCCAnalysisManager CGAM(DebugPassManager); 763 ModuleAnalysisManager MAM(DebugPassManager); 764 765 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); 766 767 Triple TargetTriple(TheModule->getTargetTriple()); 768 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple)); 769 if (DisableSimplifyLibCalls) 770 TLII->disableAllFunctions(); 771 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); 772 773 PB.registerModuleAnalyses(MAM); 774 PB.registerCGSCCAnalyses(CGAM); 775 PB.registerFunctionAnalyses(FAM); 776 PB.registerLoopAnalyses(LAM); 777 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); 778 779 // We manually collect pipeline callbacks so we can apply them at O0, where the 780 // PassBuilder does not create a pipeline. 781 std::vector<std::function<void(ModulePassManager &)>> PipelineStartEPCallbacks; 782 std::vector<std::function<void(FunctionPassManager &, PassBuilder::OptimizationLevel)>> 783 OptimizerLastEPCallbacks; 784 785 if (VerifyIR) { 786 PipelineStartEPCallbacks.push_back([VerifyIR](ModulePassManager &MPM) { 787 MPM.addPass(VerifierPass()); 788 }); 789 } 790 791 if (SanitizerOptions) { 792 if (SanitizerOptions->SanitizeMemory) { 793 MemorySanitizerOptions Options( 794 SanitizerOptions->SanitizeMemoryTrackOrigins, 795 SanitizerOptions->SanitizeRecover, 796 /*CompileKernel=*/false); 797#if LLVM_VERSION_GE(10, 0) 798 PipelineStartEPCallbacks.push_back([Options](ModulePassManager &MPM) { 799 MPM.addPass(MemorySanitizerPass(Options)); 800 }); 801#endif 802 OptimizerLastEPCallbacks.push_back( 803 [Options](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { 804 FPM.addPass(MemorySanitizerPass(Options)); 805 } 806 ); 807 } 808 809 if (SanitizerOptions->SanitizeThread) { 810#if LLVM_VERSION_GE(10, 0) 811 PipelineStartEPCallbacks.push_back([](ModulePassManager &MPM) { 812 MPM.addPass(ThreadSanitizerPass()); 813 }); 814#endif 815 OptimizerLastEPCallbacks.push_back( 816 [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { 817 FPM.addPass(ThreadSanitizerPass()); 818 } 819 ); 820 } 821 822 if (SanitizerOptions->SanitizeAddress) { 823 PipelineStartEPCallbacks.push_back([&](ModulePassManager &MPM) { 824 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>()); 825 }); 826 OptimizerLastEPCallbacks.push_back( 827 [SanitizerOptions](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { 828 FPM.addPass(AddressSanitizerPass( 829 /*CompileKernel=*/false, SanitizerOptions->SanitizeRecover, 830 /*UseAfterScope=*/true)); 831 } 832 ); 833 PipelineStartEPCallbacks.push_back( 834 [SanitizerOptions](ModulePassManager &MPM) { 835 MPM.addPass(ModuleAddressSanitizerPass( 836 /*CompileKernel=*/false, SanitizerOptions->SanitizeRecover)); 837 } 838 ); 839 } 840 } 841 842 ModulePassManager MPM(DebugPassManager); 843 if (!NoPrepopulatePasses) { 844 if (OptLevel == PassBuilder::O0) { 845 for (const auto &C : PipelineStartEPCallbacks) 846 C(MPM); 847 848 if (!OptimizerLastEPCallbacks.empty()) { 849 FunctionPassManager FPM(DebugPassManager); 850 for (const auto &C : OptimizerLastEPCallbacks) 851 C(FPM, OptLevel); 852 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 853 } 854 855 MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/false)); 856 857#if LLVM_VERSION_GE(10, 0) 858 if (PGOOpt) { 859 PB.addPGOInstrPassesForO0( 860 MPM, DebugPassManager, PGOOpt->Action == PGOOptions::IRInstr, 861 /*IsCS=*/false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile); 862 } 863#endif 864 } else { 865 for (const auto &C : PipelineStartEPCallbacks) 866 PB.registerPipelineStartEPCallback(C); 867 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) { 868 for (const auto &C : OptimizerLastEPCallbacks) 869 PB.registerOptimizerLastEPCallback(C); 870 } 871 872 switch (OptStage) { 873 case LLVMRustOptStage::PreLinkNoLTO: 874 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager); 875 break; 876 case LLVMRustOptStage::PreLinkThinLTO: 877 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager); 878 if (!OptimizerLastEPCallbacks.empty()) { 879 FunctionPassManager FPM(DebugPassManager); 880 for (const auto &C : OptimizerLastEPCallbacks) 881 C(FPM, OptLevel); 882 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 883 } 884 break; 885 case LLVMRustOptStage::PreLinkFatLTO: 886 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager); 887 break; 888 case LLVMRustOptStage::ThinLTO: 889 // FIXME: Does it make sense to pass the ModuleSummaryIndex? 890 // It only seems to be needed for C++ specific optimizations. 891 MPM = PB.buildThinLTODefaultPipeline(OptLevel, DebugPassManager, nullptr); 892 break; 893 case LLVMRustOptStage::FatLTO: 894 MPM = PB.buildLTODefaultPipeline(OptLevel, DebugPassManager, nullptr); 895 break; 896 } 897 } 898 } 899 900 if (UseThinLTOBuffers) { 901 MPM.addPass(CanonicalizeAliasesPass()); 902 MPM.addPass(NameAnonGlobalPass()); 903 } 904 905 // Upgrade all calls to old intrinsics first. 906 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;) 907 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove 908 909 MPM.run(*TheModule, MAM); 910#else 911 // The new pass manager has been available for a long time, 912 // but we don't bother supporting it on old LLVM versions. 913 report_fatal_error("New pass manager only supported since LLVM 9"); 914#endif 915} 916 917// Callback to demangle function name 918// Parameters: 919// * name to be demangled 920// * name len 921// * output buffer 922// * output buffer len 923// Returns len of demangled string, or 0 if demangle failed. 924typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t); 925 926 927namespace { 928 929class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter { 930 DemangleFn Demangle; 931 std::vector<char> Buf; 932 933public: 934 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {} 935 936 // Return empty string if demangle failed 937 // or if name does not need to be demangled 938 StringRef CallDemangle(StringRef name) { 939 if (!Demangle) { 940 return StringRef(); 941 } 942 943 if (Buf.size() < name.size() * 2) { 944 // Semangled name usually shorter than mangled, 945 // but allocate twice as much memory just in case 946 Buf.resize(name.size() * 2); 947 } 948 949 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size()); 950 if (!R) { 951 // Demangle failed. 952 return StringRef(); 953 } 954 955 auto Demangled = StringRef(Buf.data(), R); 956 if (Demangled == name) { 957 // Do not print anything if demangled name is equal to mangled. 958 return StringRef(); 959 } 960 961 return Demangled; 962 } 963 964 void emitFunctionAnnot(const Function *F, 965 formatted_raw_ostream &OS) override { 966 StringRef Demangled = CallDemangle(F->getName()); 967 if (Demangled.empty()) { 968 return; 969 } 970 971 OS << "; " << Demangled << "\n"; 972 } 973 974 void emitInstructionAnnot(const Instruction *I, 975 formatted_raw_ostream &OS) override { 976 const char *Name; 977 const Value *Value; 978 if (const CallInst *CI = dyn_cast<CallInst>(I)) { 979 Name = "call"; 980 Value = CI->getCalledValue(); 981 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) { 982 Name = "invoke"; 983 Value = II->getCalledValue(); 984 } else { 985 // Could demangle more operations, e. g. 986 // `store %place, @function`. 987 return; 988 } 989 990 if (!Value->hasName()) { 991 return; 992 } 993 994 StringRef Demangled = CallDemangle(Value->getName()); 995 if (Demangled.empty()) { 996 return; 997 } 998 999 OS << "; " << Name << " " << Demangled << "\n"; 1000 } 1001}; 1002 1003} // namespace 1004 1005extern "C" LLVMRustResult 1006LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) { 1007 std::string ErrorInfo; 1008 std::error_code EC; 1009 raw_fd_ostream OS(Path, EC, sys::fs::F_None); 1010 if (EC) 1011 ErrorInfo = EC.message(); 1012 if (ErrorInfo != "") { 1013 LLVMRustSetLastError(ErrorInfo.c_str()); 1014 return LLVMRustResult::Failure; 1015 } 1016 1017 RustAssemblyAnnotationWriter AAW(Demangle); 1018 formatted_raw_ostream FOS(OS); 1019 unwrap(M)->print(FOS, &AAW); 1020 1021 return LLVMRustResult::Success; 1022} 1023 1024extern "C" void LLVMRustPrintPasses() { 1025 LLVMInitializePasses(); 1026 struct MyListener : PassRegistrationListener { 1027 void passEnumerate(const PassInfo *Info) { 1028 StringRef PassArg = Info->getPassArgument(); 1029 StringRef PassName = Info->getPassName(); 1030 if (!PassArg.empty()) { 1031 // These unsigned->signed casts could theoretically overflow, but 1032 // realistically never will (and even if, the result is implementation 1033 // defined rather plain UB). 1034 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(), 1035 (int)PassName.size(), PassName.data()); 1036 } 1037 } 1038 } Listener; 1039 1040 PassRegistry *PR = PassRegistry::getPassRegistry(); 1041 PR->enumerateWith(&Listener); 1042} 1043 1044extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR, 1045 bool AddLifetimes) { 1046 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes); 1047} 1048 1049extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, 1050 size_t Len) { 1051 llvm::legacy::PassManager passes; 1052 1053 auto PreserveFunctions = [=](const GlobalValue &GV) { 1054 for (size_t I = 0; I < Len; I++) { 1055 if (GV.getName() == Symbols[I]) { 1056 return true; 1057 } 1058 } 1059 return false; 1060 }; 1061 1062 passes.add(llvm::createInternalizePass(PreserveFunctions)); 1063 1064 passes.run(*unwrap(M)); 1065} 1066 1067extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) { 1068 for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E; 1069 ++GV) { 1070 GV->setDoesNotThrow(); 1071 Function *F = dyn_cast<Function>(GV); 1072 if (F == nullptr) 1073 continue; 1074 1075 for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) { 1076 for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) { 1077 if (isa<InvokeInst>(I)) { 1078 InvokeInst *CI = cast<InvokeInst>(I); 1079 CI->setDoesNotThrow(); 1080 } 1081 } 1082 } 1083 } 1084} 1085 1086extern "C" void 1087LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module, 1088 LLVMTargetMachineRef TMR) { 1089 TargetMachine *Target = unwrap(TMR); 1090 unwrap(Module)->setDataLayout(Target->createDataLayout()); 1091} 1092 1093extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) { 1094 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC); 1095} 1096 1097extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { 1098 unwrap(M)->setPIELevel(PIELevel::Level::Large); 1099} 1100 1101// Here you'll find an implementation of ThinLTO as used by the Rust compiler 1102// right now. This ThinLTO support is only enabled on "recent ish" versions of 1103// LLVM, and otherwise it's just blanket rejected from other compilers. 1104// 1105// Most of this implementation is straight copied from LLVM. At the time of 1106// this writing it wasn't *quite* suitable to reuse more code from upstream 1107// for our purposes, but we should strive to upstream this support once it's 1108// ready to go! I figure we may want a bit of testing locally first before 1109// sending this upstream to LLVM. I hear though they're quite eager to receive 1110// feedback like this! 1111// 1112// If you're reading this code and wondering "what in the world" or you're 1113// working "good lord by LLVM upgrade is *still* failing due to these bindings" 1114// then fear not! (ok maybe fear a little). All code here is mostly based 1115// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM. 1116// 1117// You'll find that the general layout here roughly corresponds to the `run` 1118// method in that file as well as `ProcessThinLTOModule`. Functions are 1119// specifically commented below as well, but if you're updating this code 1120// or otherwise trying to understand it, the LLVM source will be useful in 1121// interpreting the mysteries within. 1122// 1123// Otherwise I'll apologize in advance, it probably requires a relatively 1124// significant investment on your part to "truly understand" what's going on 1125// here. Not saying I do myself, but it took me awhile staring at LLVM's source 1126// and various online resources about ThinLTO to make heads or tails of all 1127// this. 1128 1129// This is a shared data structure which *must* be threadsafe to share 1130// read-only amongst threads. This also corresponds basically to the arguments 1131// of the `ProcessThinLTOModule` function in the LLVM source. 1132struct LLVMRustThinLTOData { 1133 // The combined index that is the global analysis over all modules we're 1134 // performing ThinLTO for. This is mostly managed by LLVM. 1135 ModuleSummaryIndex Index; 1136 1137 // All modules we may look at, stored as in-memory serialized versions. This 1138 // is later used when inlining to ensure we can extract any module to inline 1139 // from. 1140 StringMap<MemoryBufferRef> ModuleMap; 1141 1142 // A set that we manage of everything we *don't* want internalized. Note that 1143 // this includes all transitive references right now as well, but it may not 1144 // always! 1145 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols; 1146 1147 // Not 100% sure what these are, but they impact what's internalized and 1148 // what's inlined across modules, I believe. 1149 StringMap<FunctionImporter::ImportMapTy> ImportLists; 1150 StringMap<FunctionImporter::ExportSetTy> ExportLists; 1151 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries; 1152 1153 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} 1154}; 1155 1156// Just an argument to the `LLVMRustCreateThinLTOData` function below. 1157struct LLVMRustThinLTOModule { 1158 const char *identifier; 1159 const char *data; 1160 size_t len; 1161}; 1162 1163// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it 1164// does. 1165static const GlobalValueSummary * 1166getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { 1167 auto StrongDefForLinker = llvm::find_if( 1168 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) { 1169 auto Linkage = Summary->linkage(); 1170 return !GlobalValue::isAvailableExternallyLinkage(Linkage) && 1171 !GlobalValue::isWeakForLinker(Linkage); 1172 }); 1173 if (StrongDefForLinker != GVSummaryList.end()) 1174 return StrongDefForLinker->get(); 1175 1176 auto FirstDefForLinker = llvm::find_if( 1177 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) { 1178 auto Linkage = Summary->linkage(); 1179 return !GlobalValue::isAvailableExternallyLinkage(Linkage); 1180 }); 1181 if (FirstDefForLinker == GVSummaryList.end()) 1182 return nullptr; 1183 return FirstDefForLinker->get(); 1184} 1185 1186// The main entry point for creating the global ThinLTO analysis. The structure 1187// here is basically the same as before threads are spawned in the `run` 1188// function of `lib/LTO/ThinLTOCodeGenerator.cpp`. 1189extern "C" LLVMRustThinLTOData* 1190LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, 1191 int num_modules, 1192 const char **preserved_symbols, 1193 int num_symbols) { 1194#if LLVM_VERSION_GE(10, 0) 1195 auto Ret = std::make_unique<LLVMRustThinLTOData>(); 1196#else 1197 auto Ret = llvm::make_unique<LLVMRustThinLTOData>(); 1198#endif 1199 1200 // Load each module's summary and merge it into one combined index 1201 for (int i = 0; i < num_modules; i++) { 1202 auto module = &modules[i]; 1203 StringRef buffer(module->data, module->len); 1204 MemoryBufferRef mem_buffer(buffer, module->identifier); 1205 1206 Ret->ModuleMap[module->identifier] = mem_buffer; 1207 1208 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { 1209 LLVMRustSetLastError(toString(std::move(Err)).c_str()); 1210 return nullptr; 1211 } 1212 } 1213 1214 // Collect for each module the list of function it defines (GUID -> Summary) 1215 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries); 1216 1217 // Convert the preserved symbols set from string to GUID, this is then needed 1218 // for internalization. 1219 for (int i = 0; i < num_symbols; i++) { 1220 auto GUID = GlobalValue::getGUID(preserved_symbols[i]); 1221 Ret->GUIDPreservedSymbols.insert(GUID); 1222 } 1223 1224 // Collect the import/export lists for all modules from the call-graph in the 1225 // combined index 1226 // 1227 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` 1228 auto deadIsPrevailing = [&](GlobalValue::GUID G) { 1229 return PrevailingType::Unknown; 1230 }; 1231 // We don't have a complete picture in our use of ThinLTO, just our immediate 1232 // crate, so we need `ImportEnabled = false` to limit internalization. 1233 // Otherwise, we sometimes lose `static` values -- see #60184. 1234 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, 1235 deadIsPrevailing, /* ImportEnabled = */ false); 1236 ComputeCrossModuleImport( 1237 Ret->Index, 1238 Ret->ModuleToDefinedGVSummaries, 1239 Ret->ImportLists, 1240 Ret->ExportLists 1241 ); 1242 1243 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it 1244 // impacts the caching. 1245 // 1246 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this 1247 // being lifted from `lib/LTO/LTO.cpp` as well 1248 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; 1249 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; 1250 for (auto &I : Ret->Index) { 1251 if (I.second.SummaryList.size() > 1) 1252 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList); 1253 } 1254 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { 1255 const auto &Prevailing = PrevailingCopy.find(GUID); 1256 if (Prevailing == PrevailingCopy.end()) 1257 return true; 1258 return Prevailing->second == S; 1259 }; 1260 auto recordNewLinkage = [&](StringRef ModuleIdentifier, 1261 GlobalValue::GUID GUID, 1262 GlobalValue::LinkageTypes NewLinkage) { 1263 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; 1264 }; 1265#if LLVM_VERSION_GE(9, 0) 1266 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage, 1267 Ret->GUIDPreservedSymbols); 1268#else 1269 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage); 1270#endif 1271 1272 // Here we calculate an `ExportedGUIDs` set for use in the `isExported` 1273 // callback below. This callback below will dictate the linkage for all 1274 // summaries in the index, and we basically just only want to ensure that dead 1275 // symbols are internalized. Otherwise everything that's already external 1276 // linkage will stay as external, and internal will stay as internal. 1277 std::set<GlobalValue::GUID> ExportedGUIDs; 1278 for (auto &List : Ret->Index) { 1279 for (auto &GVS: List.second.SummaryList) { 1280 if (GlobalValue::isLocalLinkage(GVS->linkage())) 1281 continue; 1282 auto GUID = GVS->getOriginalName(); 1283 if (GVS->flags().Live) 1284 ExportedGUIDs.insert(GUID); 1285 } 1286 } 1287#if LLVM_VERSION_GE(10, 0) 1288 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) { 1289 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); 1290 return (ExportList != Ret->ExportLists.end() && 1291 ExportList->second.count(VI)) || 1292 ExportedGUIDs.count(VI.getGUID()); 1293 }; 1294 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing); 1295#else 1296 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) { 1297 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); 1298 return (ExportList != Ret->ExportLists.end() && 1299 ExportList->second.count(GUID)) || 1300 ExportedGUIDs.count(GUID); 1301 }; 1302 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported); 1303#endif 1304 1305 return Ret.release(); 1306} 1307 1308extern "C" void 1309LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { 1310 delete Data; 1311} 1312 1313// Below are the various passes that happen *per module* when doing ThinLTO. 1314// 1315// In other words, these are the functions that are all run concurrently 1316// with one another, one per module. The passes here correspond to the analysis 1317// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the 1318// `ProcessThinLTOModule` function. Here they're split up into separate steps 1319// so rustc can save off the intermediate bytecode between each step. 1320 1321extern "C" bool 1322LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 1323 Module &Mod = *unwrap(M); 1324 if (renameModuleForThinLTO(Mod, Data->Index)) { 1325 LLVMRustSetLastError("renameModuleForThinLTO failed"); 1326 return false; 1327 } 1328 return true; 1329} 1330 1331extern "C" bool 1332LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 1333 Module &Mod = *unwrap(M); 1334 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); 1335 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals); 1336 return true; 1337} 1338 1339extern "C" bool 1340LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 1341 Module &Mod = *unwrap(M); 1342 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); 1343 thinLTOInternalizeModule(Mod, DefinedGlobals); 1344 return true; 1345} 1346 1347extern "C" bool 1348LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 1349 Module &Mod = *unwrap(M); 1350 1351 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier()); 1352 auto Loader = [&](StringRef Identifier) { 1353 const auto &Memory = Data->ModuleMap.lookup(Identifier); 1354 auto &Context = Mod.getContext(); 1355 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true); 1356 1357 if (!MOrErr) 1358 return MOrErr; 1359 1360 // The rest of this closure is a workaround for 1361 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports 1362 // we accidentally import wasm custom sections into different modules, 1363 // duplicating them by in the final output artifact. 1364 // 1365 // The issue is worked around here by manually removing the 1366 // `wasm.custom_sections` named metadata node from any imported module. This 1367 // we know isn't used by any optimization pass so there's no need for it to 1368 // be imported. 1369 // 1370 // Note that the metadata is currently lazily loaded, so we materialize it 1371 // here before looking up if there's metadata inside. The `FunctionImporter` 1372 // will immediately materialize metadata anyway after an import, so this 1373 // shouldn't be a perf hit. 1374 if (Error Err = (*MOrErr)->materializeMetadata()) { 1375 Expected<std::unique_ptr<Module>> Ret(std::move(Err)); 1376 return Ret; 1377 } 1378 1379 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections"); 1380 if (WasmCustomSections) 1381 WasmCustomSections->eraseFromParent(); 1382 1383 return MOrErr; 1384 }; 1385 FunctionImporter Importer(Data->Index, Loader); 1386 Expected<bool> Result = Importer.importFunctions(Mod, ImportList); 1387 if (!Result) { 1388 LLVMRustSetLastError(toString(Result.takeError()).c_str()); 1389 return false; 1390 } 1391 return true; 1392} 1393 1394extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload 1395 const char*, // importing module name 1396 const char*); // imported module name 1397 1398// Calls `module_name_callback` for each module import done by ThinLTO. 1399// The callback is provided with regular null-terminated C strings. 1400extern "C" void 1401LLVMRustGetThinLTOModuleImports(const LLVMRustThinLTOData *data, 1402 LLVMRustModuleNameCallback module_name_callback, 1403 void* callback_payload) { 1404 for (const auto& importing_module : data->ImportLists) { 1405 const std::string importing_module_id = importing_module.getKey().str(); 1406 const auto& imports = importing_module.getValue(); 1407 for (const auto& imported_module : imports) { 1408 const std::string imported_module_id = imported_module.getKey().str(); 1409 module_name_callback(callback_payload, 1410 importing_module_id.c_str(), 1411 imported_module_id.c_str()); 1412 } 1413 } 1414} 1415 1416// This struct and various functions are sort of a hack right now, but the 1417// problem is that we've got in-memory LLVM modules after we generate and 1418// optimize all codegen-units for one compilation in rustc. To be compatible 1419// with the LTO support above we need to serialize the modules plus their 1420// ThinLTO summary into memory. 1421// 1422// This structure is basically an owned version of a serialize module, with 1423// a ThinLTO summary attached. 1424struct LLVMRustThinLTOBuffer { 1425 std::string data; 1426}; 1427 1428extern "C" LLVMRustThinLTOBuffer* 1429LLVMRustThinLTOBufferCreate(LLVMModuleRef M) { 1430#if LLVM_VERSION_GE(10, 0) 1431 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>(); 1432#else 1433 auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>(); 1434#endif 1435 { 1436 raw_string_ostream OS(Ret->data); 1437 { 1438 legacy::PassManager PM; 1439 PM.add(createWriteThinLTOBitcodePass(OS)); 1440 PM.run(*unwrap(M)); 1441 } 1442 } 1443 return Ret.release(); 1444} 1445 1446extern "C" void 1447LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { 1448 delete Buffer; 1449} 1450 1451extern "C" const void* 1452LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { 1453 return Buffer->data.data(); 1454} 1455 1456extern "C" size_t 1457LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { 1458 return Buffer->data.length(); 1459} 1460 1461// This is what we used to parse upstream bitcode for actual ThinLTO 1462// processing. We'll call this once per module optimized through ThinLTO, and 1463// it'll be called concurrently on many threads. 1464extern "C" LLVMModuleRef 1465LLVMRustParseBitcodeForLTO(LLVMContextRef Context, 1466 const char *data, 1467 size_t len, 1468 const char *identifier) { 1469 StringRef Data(data, len); 1470 MemoryBufferRef Buffer(Data, identifier); 1471 unwrap(Context)->enableDebugTypeODRUniquing(); 1472 Expected<std::unique_ptr<Module>> SrcOrError = 1473 parseBitcodeFile(Buffer, *unwrap(Context)); 1474 if (!SrcOrError) { 1475 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); 1476 return nullptr; 1477 } 1478 return wrap(std::move(*SrcOrError).release()); 1479} 1480 1481// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See 1482// the comment in `back/lto.rs` for why this exists. 1483extern "C" void 1484LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod, 1485 DICompileUnit **A, 1486 DICompileUnit **B) { 1487 Module *M = unwrap(Mod); 1488 DICompileUnit **Cur = A; 1489 DICompileUnit **Next = B; 1490 for (DICompileUnit *CU : M->debug_compile_units()) { 1491 *Cur = CU; 1492 Cur = Next; 1493 Next = nullptr; 1494 if (Cur == nullptr) 1495 break; 1496 } 1497} 1498 1499// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See 1500// the comment in `back/lto.rs` for why this exists. 1501extern "C" void 1502LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) { 1503 Module *M = unwrap(Mod); 1504 1505 // If the original source module didn't have a `DICompileUnit` then try to 1506 // merge all the existing compile units. If there aren't actually any though 1507 // then there's not much for us to do so return. 1508 if (Unit == nullptr) { 1509 for (DICompileUnit *CU : M->debug_compile_units()) { 1510 Unit = CU; 1511 break; 1512 } 1513 if (Unit == nullptr) 1514 return; 1515 } 1516 1517 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and 1518 // process it recursively. Note that we specifically iterate over instructions 1519 // to ensure we feed everything into it. 1520 DebugInfoFinder Finder; 1521 Finder.processModule(*M); 1522 for (Function &F : M->functions()) { 1523 for (auto &FI : F) { 1524 for (Instruction &BI : FI) { 1525 if (auto Loc = BI.getDebugLoc()) 1526 Finder.processLocation(*M, Loc); 1527 if (auto DVI = dyn_cast<DbgValueInst>(&BI)) 1528 Finder.processValue(*M, DVI); 1529 if (auto DDI = dyn_cast<DbgDeclareInst>(&BI)) 1530 Finder.processDeclare(*M, DDI); 1531 } 1532 } 1533 } 1534 1535 // After we've found all our debuginfo, rewrite all subprograms to point to 1536 // the same `DICompileUnit`. 1537 for (auto &F : Finder.subprograms()) { 1538 F->replaceUnit(Unit); 1539 } 1540 1541 // Erase any other references to other `DICompileUnit` instances, the verifier 1542 // will later ensure that we don't actually have any other stale references to 1543 // worry about. 1544 auto *MD = M->getNamedMetadata("llvm.dbg.cu"); 1545 MD->clearOperands(); 1546 MD->addOperand(Unit); 1547}