fork/rust
git clone https://git.julia.blue/fork/rust
8d69840ab92
1.43.1
1#include <stdio.h> 2 3#include <vector> 4#include <set> 5 6#include "rustllvm.h" 7 8#include "llvm/Analysis/TargetLibraryInfo.h" 9#include "llvm/Analysis/TargetTransformInfo.h" 10#include "llvm/CodeGen/TargetSubtargetInfo.h" 11#include "llvm/InitializePasses.h" 12#include "llvm/IR/AutoUpgrade.h" 13#include "llvm/IR/AssemblyAnnotationWriter.h" 14#include "llvm/IR/IntrinsicInst.h" 15#include "llvm/IR/Verifier.h" 16#include "llvm/Passes/PassBuilder.h" 17#if LLVM_VERSION_GE(9, 0) 18#include "llvm/Passes/StandardInstrumentations.h" 19#endif 20#include "llvm/Support/CBindingWrapping.h" 21#include "llvm/Support/FileSystem.h" 22#include "llvm/Support/Host.h" 23#include "llvm/Target/TargetMachine.h" 24#include "llvm/Transforms/IPO/PassManagerBuilder.h" 25#include "llvm/Transforms/IPO/AlwaysInliner.h" 26#include "llvm/Transforms/IPO/FunctionImport.h" 27#include "llvm/Transforms/Utils/FunctionImportUtils.h" 28#include "llvm/LTO/LTO.h" 29#include "llvm-c/Transforms/PassManagerBuilder.h" 30 31#include "llvm/Transforms/Instrumentation.h" 32#if LLVM_VERSION_GE(9, 0) 33#include "llvm/Transforms/Instrumentation/AddressSanitizer.h" 34#include "llvm/Support/TimeProfiler.h" 35#endif 36#if LLVM_VERSION_GE(8, 0) 37#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" 38#include "llvm/Transforms/Instrumentation/MemorySanitizer.h" 39#endif 40#if LLVM_VERSION_GE(9, 0) 41#include "llvm/Transforms/Utils/CanonicalizeAliases.h" 42#endif 43#include "llvm/Transforms/Utils/NameAnonGlobals.h" 44 45using namespace llvm; 46 47typedef struct LLVMOpaquePass *LLVMPassRef; 48typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; 49 50DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef) 51DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) 52DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder, 53 LLVMPassManagerBuilderRef) 54 55extern "C" void LLVMInitializePasses() { 56 PassRegistry &Registry = *PassRegistry::getPassRegistry(); 57 initializeCore(Registry); 58 initializeCodeGen(Registry); 59 initializeScalarOpts(Registry); 60 initializeVectorization(Registry); 61 initializeIPO(Registry); 62 initializeAnalysis(Registry); 63 initializeTransformUtils(Registry); 64 initializeInstCombine(Registry); 65 initializeInstrumentation(Registry); 66 initializeTarget(Registry); 67} 68 69extern "C" void LLVMTimeTraceProfilerInitialize() { 70#if LLVM_VERSION_GE(9, 0) 71 timeTraceProfilerInitialize(); 72#endif 73} 74 75extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) { 76#if LLVM_VERSION_GE(9, 0) 77 StringRef FN(FileName); 78 std::error_code EC; 79 raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways); 80 81 timeTraceProfilerWrite(OS); 82 timeTraceProfilerCleanup(); 83#endif 84} 85 86enum class LLVMRustPassKind { 87 Other, 88 Function, 89 Module, 90}; 91 92static LLVMRustPassKind toRust(PassKind Kind) { 93 switch (Kind) { 94 case PT_Function: 95 return LLVMRustPassKind::Function; 96 case PT_Module: 97 return LLVMRustPassKind::Module; 98 default: 99 return LLVMRustPassKind::Other; 100 } 101} 102 103extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) { 104 StringRef SR(PassName); 105 PassRegistry *PR = PassRegistry::getPassRegistry(); 106 107 const PassInfo *PI = PR->getPassInfo(SR); 108 if (PI) { 109 return wrap(PI->createPass()); 110 } 111 return nullptr; 112} 113 114extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) { 115 const bool CompileKernel = false; 116 const bool UseAfterScope = true; 117 118 return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope)); 119} 120 121extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) { 122 const bool CompileKernel = false; 123 124#if LLVM_VERSION_GE(9, 0) 125 return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover)); 126#else 127 return wrap(createAddressSanitizerModulePass(CompileKernel, Recover)); 128#endif 129} 130 131extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) { 132#if LLVM_VERSION_GE(9, 0) 133 const bool CompileKernel = false; 134 135 return wrap(createMemorySanitizerLegacyPassPass( 136 MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel})); 137#elif LLVM_VERSION_GE(8, 0) 138 return wrap(createMemorySanitizerLegacyPassPass(TrackOrigins, Recover)); 139#else 140 return wrap(createMemorySanitizerPass(TrackOrigins, Recover)); 141#endif 142} 143 144extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() { 145#if LLVM_VERSION_GE(8, 0) 146 return wrap(createThreadSanitizerLegacyPassPass()); 147#else 148 return wrap(createThreadSanitizerPass()); 149#endif 150} 151 152extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) { 153 assert(RustPass); 154 Pass *Pass = unwrap(RustPass); 155 return toRust(Pass->getPassKind()); 156} 157 158extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) { 159 assert(RustPass); 160 Pass *Pass = unwrap(RustPass); 161 PassManagerBase *PMB = unwrap(PMR); 162 PMB->add(Pass); 163} 164 165extern "C" 166void LLVMRustPassManagerBuilderPopulateThinLTOPassManager( 167 LLVMPassManagerBuilderRef PMBR, 168 LLVMPassManagerRef PMR 169) { 170 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR)); 171} 172 173extern "C" 174void LLVMRustAddLastExtensionPasses( 175 LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) { 176 auto AddExtensionPasses = [Passes, NumPasses]( 177 const PassManagerBuilder &Builder, PassManagerBase &PM) { 178 for (size_t I = 0; I < NumPasses; I++) { 179 PM.add(unwrap(Passes[I])); 180 } 181 }; 182 // Add the passes to both of the pre-finalization extension points, 183 // so they are run for optimized and non-optimized builds. 184 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast, 185 AddExtensionPasses); 186 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, 187 AddExtensionPasses); 188} 189 190#ifdef LLVM_COMPONENT_X86 191#define SUBTARGET_X86 SUBTARGET(X86) 192#else 193#define SUBTARGET_X86 194#endif 195 196#ifdef LLVM_COMPONENT_ARM 197#define SUBTARGET_ARM SUBTARGET(ARM) 198#else 199#define SUBTARGET_ARM 200#endif 201 202#ifdef LLVM_COMPONENT_AARCH64 203#define SUBTARGET_AARCH64 SUBTARGET(AArch64) 204#else 205#define SUBTARGET_AARCH64 206#endif 207 208#ifdef LLVM_COMPONENT_MIPS 209#define SUBTARGET_MIPS SUBTARGET(Mips) 210#else 211#define SUBTARGET_MIPS 212#endif 213 214#ifdef LLVM_COMPONENT_POWERPC 215#define SUBTARGET_PPC SUBTARGET(PPC) 216#else 217#define SUBTARGET_PPC 218#endif 219 220#ifdef LLVM_COMPONENT_SYSTEMZ 221#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ) 222#else 223#define SUBTARGET_SYSTEMZ 224#endif 225 226#ifdef LLVM_COMPONENT_MSP430 227#define SUBTARGET_MSP430 SUBTARGET(MSP430) 228#else 229#define SUBTARGET_MSP430 230#endif 231 232#ifdef LLVM_COMPONENT_RISCV 233#define SUBTARGET_RISCV SUBTARGET(RISCV) 234#else 235#define SUBTARGET_RISCV 236#endif 237 238#ifdef LLVM_COMPONENT_SPARC 239#define SUBTARGET_SPARC SUBTARGET(Sparc) 240#else 241#define SUBTARGET_SPARC 242#endif 243 244#ifdef LLVM_COMPONENT_HEXAGON 245#define SUBTARGET_HEXAGON SUBTARGET(Hexagon) 246#else 247#define SUBTARGET_HEXAGON 248#endif 249 250#define GEN_SUBTARGETS \ 251 SUBTARGET_X86 \ 252 SUBTARGET_ARM \ 253 SUBTARGET_AARCH64 \ 254 SUBTARGET_MIPS \ 255 SUBTARGET_PPC \ 256 SUBTARGET_SYSTEMZ \ 257 SUBTARGET_MSP430 \ 258 SUBTARGET_SPARC \ 259 SUBTARGET_HEXAGON \ 260 SUBTARGET_RISCV \ 261 262#define SUBTARGET(x) \ 263 namespace llvm { \ 264 extern const SubtargetFeatureKV x##FeatureKV[]; \ 265 extern const SubtargetFeatureKV x##SubTypeKV[]; \ 266 } 267 268GEN_SUBTARGETS 269#undef SUBTARGET 270 271extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, 272 const char *Feature) { 273 TargetMachine *Target = unwrap(TM); 274 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 275 return MCInfo->checkFeatures(std::string("+") + Feature); 276} 277 278enum class LLVMRustCodeModel { 279 Other, 280 Small, 281 Kernel, 282 Medium, 283 Large, 284 None, 285}; 286 287static CodeModel::Model fromRust(LLVMRustCodeModel Model) { 288 switch (Model) { 289 case LLVMRustCodeModel::Small: 290 return CodeModel::Small; 291 case LLVMRustCodeModel::Kernel: 292 return CodeModel::Kernel; 293 case LLVMRustCodeModel::Medium: 294 return CodeModel::Medium; 295 case LLVMRustCodeModel::Large: 296 return CodeModel::Large; 297 default: 298 report_fatal_error("Bad CodeModel."); 299 } 300} 301 302enum class LLVMRustCodeGenOptLevel { 303 Other, 304 None, 305 Less, 306 Default, 307 Aggressive, 308}; 309 310static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) { 311 switch (Level) { 312 case LLVMRustCodeGenOptLevel::None: 313 return CodeGenOpt::None; 314 case LLVMRustCodeGenOptLevel::Less: 315 return CodeGenOpt::Less; 316 case LLVMRustCodeGenOptLevel::Default: 317 return CodeGenOpt::Default; 318 case LLVMRustCodeGenOptLevel::Aggressive: 319 return CodeGenOpt::Aggressive; 320 default: 321 report_fatal_error("Bad CodeGenOptLevel."); 322 } 323} 324 325enum class LLVMRustPassBuilderOptLevel { 326 O0, 327 O1, 328 O2, 329 O3, 330 Os, 331 Oz, 332}; 333 334static PassBuilder::OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) { 335 switch (Level) { 336 case LLVMRustPassBuilderOptLevel::O0: 337 return PassBuilder::O0; 338 case LLVMRustPassBuilderOptLevel::O1: 339 return PassBuilder::O1; 340 case LLVMRustPassBuilderOptLevel::O2: 341 return PassBuilder::O2; 342 case LLVMRustPassBuilderOptLevel::O3: 343 return PassBuilder::O3; 344 case LLVMRustPassBuilderOptLevel::Os: 345 return PassBuilder::Os; 346 case LLVMRustPassBuilderOptLevel::Oz: 347 return PassBuilder::Oz; 348 default: 349 report_fatal_error("Bad PassBuilderOptLevel."); 350 } 351} 352 353enum class LLVMRustRelocMode { 354 Default, 355 Static, 356 PIC, 357 DynamicNoPic, 358 ROPI, 359 RWPI, 360 ROPIRWPI, 361}; 362 363static Optional<Reloc::Model> fromRust(LLVMRustRelocMode RustReloc) { 364 switch (RustReloc) { 365 case LLVMRustRelocMode::Default: 366 return None; 367 case LLVMRustRelocMode::Static: 368 return Reloc::Static; 369 case LLVMRustRelocMode::PIC: 370 return Reloc::PIC_; 371 case LLVMRustRelocMode::DynamicNoPic: 372 return Reloc::DynamicNoPIC; 373 case LLVMRustRelocMode::ROPI: 374 return Reloc::ROPI; 375 case LLVMRustRelocMode::RWPI: 376 return Reloc::RWPI; 377 case LLVMRustRelocMode::ROPIRWPI: 378 return Reloc::ROPI_RWPI; 379 } 380 report_fatal_error("Bad RelocModel."); 381} 382 383#ifdef LLVM_RUSTLLVM 384/// getLongestEntryLength - Return the length of the longest entry in the table. 385template<typename KV> 386static size_t getLongestEntryLength(ArrayRef<KV> Table) { 387 size_t MaxLen = 0; 388 for (auto &I : Table) 389 MaxLen = std::max(MaxLen, std::strlen(I.Key)); 390 return MaxLen; 391} 392 393extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) { 394 const TargetMachine *Target = unwrap(TM); 395 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 396 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch(); 397 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); 398 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable(); 399 unsigned MaxCPULen = getLongestEntryLength(CPUTable); 400 401 printf("Available CPUs for this target:\n"); 402 if (HostArch == TargetArch) { 403 const StringRef HostCPU = sys::getHostCPUName(); 404 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n", 405 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data()); 406 } 407 for (auto &CPU : CPUTable) 408 printf(" %-*s\n", MaxCPULen, CPU.Key); 409 printf("\n"); 410} 411 412extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) { 413 const TargetMachine *Target = unwrap(TM); 414 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 415 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable(); 416 unsigned MaxFeatLen = getLongestEntryLength(FeatTable); 417 418 printf("Available features for this target:\n"); 419 for (auto &Feature : FeatTable) 420 printf(" %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc); 421 printf("\n"); 422 423 printf("Use +feature to enable a feature, or -feature to disable it.\n" 424 "For example, rustc -C -target-cpu=mycpu -C " 425 "target-feature=+feature1,-feature2\n\n"); 426} 427 428#else 429 430extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) { 431 printf("Target CPU help is not supported by this LLVM version.\n\n"); 432} 433 434extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) { 435 printf("Target features help is not supported by this LLVM version.\n\n"); 436} 437#endif 438 439extern "C" const char* LLVMRustGetHostCPUName(size_t *len) { 440 StringRef Name = sys::getHostCPUName(); 441 *len = Name.size(); 442 return Name.data(); 443} 444 445extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( 446 const char *TripleStr, const char *CPU, const char *Feature, 447 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocMode RustReloc, 448 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, 449 bool PositionIndependentExecutable, bool FunctionSections, 450 bool DataSections, 451 bool TrapUnreachable, 452 bool Singlethread, 453 bool AsmComments, 454 bool EmitStackSizeSection, 455 bool RelaxELFRelocations) { 456 457 auto OptLevel = fromRust(RustOptLevel); 458 auto RM = fromRust(RustReloc); 459 460 std::string Error; 461 Triple Trip(Triple::normalize(TripleStr)); 462 const llvm::Target *TheTarget = 463 TargetRegistry::lookupTarget(Trip.getTriple(), Error); 464 if (TheTarget == nullptr) { 465 LLVMRustSetLastError(Error.c_str()); 466 return nullptr; 467 } 468 469 TargetOptions Options; 470 471 Options.FloatABIType = FloatABI::Default; 472 if (UseSoftFloat) { 473 Options.FloatABIType = FloatABI::Soft; 474 } 475 Options.DataSections = DataSections; 476 Options.FunctionSections = FunctionSections; 477 Options.MCOptions.AsmVerbose = AsmComments; 478 Options.MCOptions.PreserveAsmComments = AsmComments; 479 Options.MCOptions.ABIName = ABIStr; 480 Options.RelaxELFRelocations = RelaxELFRelocations; 481 482 if (TrapUnreachable) { 483 // Tell LLVM to codegen `unreachable` into an explicit trap instruction. 484 // This limits the extent of possible undefined behavior in some cases, as 485 // it prevents control flow from "falling through" into whatever code 486 // happens to be laid out next in memory. 487 Options.TrapUnreachable = true; 488 } 489 490 if (Singlethread) { 491 Options.ThreadModel = ThreadModel::Single; 492 } 493 494 Options.EmitStackSizeSection = EmitStackSizeSection; 495 496 Optional<CodeModel::Model> CM; 497 if (RustCM != LLVMRustCodeModel::None) 498 CM = fromRust(RustCM); 499 TargetMachine *TM = TheTarget->createTargetMachine( 500 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); 501 return wrap(TM); 502} 503 504extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { 505 delete unwrap(TM); 506} 507 508extern "C" void LLVMRustConfigurePassManagerBuilder( 509 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel, 510 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO, 511 const char* PGOGenPath, const char* PGOUsePath) { 512 unwrap(PMBR)->MergeFunctions = MergeFunctions; 513 unwrap(PMBR)->SLPVectorize = SLPVectorize; 514 unwrap(PMBR)->OptLevel = fromRust(OptLevel); 515 unwrap(PMBR)->LoopVectorize = LoopVectorize; 516 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO; 517 518 if (PGOGenPath) { 519 assert(!PGOUsePath); 520 unwrap(PMBR)->EnablePGOInstrGen = true; 521 unwrap(PMBR)->PGOInstrGen = PGOGenPath; 522 } 523 if (PGOUsePath) { 524 assert(!PGOGenPath); 525 unwrap(PMBR)->PGOInstrUse = PGOUsePath; 526 } 527} 528 529// Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo` 530// field of a PassManagerBuilder, we expose our own method of doing so. 531extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR, 532 LLVMModuleRef M, 533 bool DisableSimplifyLibCalls) { 534 Triple TargetTriple(unwrap(M)->getTargetTriple()); 535 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple); 536 if (DisableSimplifyLibCalls) 537 TLI->disableAllFunctions(); 538 unwrap(PMBR)->LibraryInfo = TLI; 539} 540 541// Unfortunately, the LLVM C API doesn't provide a way to create the 542// TargetLibraryInfo pass, so we use this method to do so. 543extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M, 544 bool DisableSimplifyLibCalls) { 545 Triple TargetTriple(unwrap(M)->getTargetTriple()); 546 TargetLibraryInfoImpl TLII(TargetTriple); 547 if (DisableSimplifyLibCalls) 548 TLII.disableAllFunctions(); 549 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII)); 550} 551 552// Unfortunately, the LLVM C API doesn't provide an easy way of iterating over 553// all the functions in a module, so we do that manually here. You'll find 554// similar code in clang's BackendUtil.cpp file. 555extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR, 556 LLVMModuleRef M) { 557 llvm::legacy::FunctionPassManager *P = 558 unwrap<llvm::legacy::FunctionPassManager>(PMR); 559 P->doInitialization(); 560 561 // Upgrade all calls to old intrinsics first. 562 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;) 563 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove 564 565 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E; 566 ++I) 567 if (!I->isDeclaration()) 568 P->run(*I); 569 570 P->doFinalization(); 571} 572 573extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) { 574 // Initializing the command-line options more than once is not allowed. So, 575 // check if they've already been initialized. (This could happen if we're 576 // being called from rustpkg, for example). If the arguments change, then 577 // that's just kinda unfortunate. 578 static bool Initialized = false; 579 if (Initialized) 580 return; 581 Initialized = true; 582 cl::ParseCommandLineOptions(Argc, Argv); 583} 584 585enum class LLVMRustFileType { 586 Other, 587 AssemblyFile, 588 ObjectFile, 589}; 590 591#if LLVM_VERSION_GE(10, 0) 592static CodeGenFileType fromRust(LLVMRustFileType Type) { 593 switch (Type) { 594 case LLVMRustFileType::AssemblyFile: 595 return CGFT_AssemblyFile; 596 case LLVMRustFileType::ObjectFile: 597 return CGFT_ObjectFile; 598 default: 599 report_fatal_error("Bad FileType."); 600 } 601} 602#else 603static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) { 604 switch (Type) { 605 case LLVMRustFileType::AssemblyFile: 606 return TargetMachine::CGFT_AssemblyFile; 607 case LLVMRustFileType::ObjectFile: 608 return TargetMachine::CGFT_ObjectFile; 609 default: 610 report_fatal_error("Bad FileType."); 611 } 612} 613#endif 614 615extern "C" LLVMRustResult 616LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, 617 LLVMModuleRef M, const char *Path, 618 LLVMRustFileType RustFileType) { 619 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR); 620 auto FileType = fromRust(RustFileType); 621 622 std::string ErrorInfo; 623 std::error_code EC; 624 raw_fd_ostream OS(Path, EC, sys::fs::F_None); 625 if (EC) 626 ErrorInfo = EC.message(); 627 if (ErrorInfo != "") { 628 LLVMRustSetLastError(ErrorInfo.c_str()); 629 return LLVMRustResult::Failure; 630 } 631 632 buffer_ostream BOS(OS); 633 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false); 634 PM->run(*unwrap(M)); 635 636 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output 637 // stream (OS), so the only real safe place to delete this is here? Don't we 638 // wish this was written in Rust? 639 LLVMDisposePassManager(PMR); 640 return LLVMRustResult::Success; 641} 642 643extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler 644 const char*, // pass name 645 const char*); // IR name 646extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler 647 648#if LLVM_VERSION_GE(9, 0) 649 650std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { 651 if (any_isa<const Module *>(WrappedIr)) 652 return any_cast<const Module *>(WrappedIr)->getName().str(); 653 if (any_isa<const Function *>(WrappedIr)) 654 return any_cast<const Function *>(WrappedIr)->getName().str(); 655 if (any_isa<const Loop *>(WrappedIr)) 656 return any_cast<const Loop *>(WrappedIr)->getName().str(); 657 if (any_isa<const LazyCallGraph::SCC *>(WrappedIr)) 658 return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName(); 659 return "<UNKNOWN>"; 660} 661 662 663void LLVMSelfProfileInitializeCallbacks( 664 PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler, 665 LLVMRustSelfProfileBeforePassCallback BeforePassCallback, 666 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { 667 PIC.registerBeforePassCallback([LlvmSelfProfiler, BeforePassCallback]( 668 StringRef Pass, llvm::Any Ir) { 669 std::string PassName = Pass.str(); 670 std::string IrName = LLVMRustwrappedIrGetName(Ir); 671 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); 672 return true; 673 }); 674 675 PIC.registerAfterPassCallback( 676 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { 677 AfterPassCallback(LlvmSelfProfiler); 678 }); 679 680 PIC.registerAfterPassInvalidatedCallback( 681 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass) { 682 AfterPassCallback(LlvmSelfProfiler); 683 }); 684 685 PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback]( 686 StringRef Pass, llvm::Any Ir) { 687 std::string PassName = Pass.str(); 688 std::string IrName = LLVMRustwrappedIrGetName(Ir); 689 BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); 690 }); 691 692 PIC.registerAfterAnalysisCallback( 693 [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { 694 AfterPassCallback(LlvmSelfProfiler); 695 }); 696} 697#endif 698 699enum class LLVMRustOptStage { 700 PreLinkNoLTO, 701 PreLinkThinLTO, 702 PreLinkFatLTO, 703 ThinLTO, 704 FatLTO, 705}; 706 707struct LLVMRustSanitizerOptions { 708 bool SanitizeMemory; 709 bool SanitizeThread; 710 bool SanitizeAddress; 711 bool SanitizeRecover; 712 int SanitizeMemoryTrackOrigins; 713}; 714 715extern "C" void 716LLVMRustOptimizeWithNewPassManager( 717 LLVMModuleRef ModuleRef, 718 LLVMTargetMachineRef TMRef, 719 LLVMRustPassBuilderOptLevel OptLevelRust, 720 LLVMRustOptStage OptStage, 721 bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers, 722 bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, 723 bool DisableSimplifyLibCalls, 724 LLVMRustSanitizerOptions *SanitizerOptions, 725 const char *PGOGenPath, const char *PGOUsePath, 726 void* LlvmSelfProfiler, 727 LLVMRustSelfProfileBeforePassCallback BeforePassCallback, 728 LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { 729#if LLVM_VERSION_GE(9, 0) 730 Module *TheModule = unwrap(ModuleRef); 731 TargetMachine *TM = unwrap(TMRef); 732 PassBuilder::OptimizationLevel OptLevel = fromRust(OptLevelRust); 733 734 // FIXME: MergeFunctions is not supported by NewPM yet. 735 (void) MergeFunctions; 736 737 PipelineTuningOptions PTO; 738 PTO.LoopUnrolling = UnrollLoops; 739 PTO.LoopInterleaving = UnrollLoops; 740 PTO.LoopVectorization = LoopVectorize; 741 PTO.SLPVectorization = SLPVectorize; 742 743 PassInstrumentationCallbacks PIC; 744 StandardInstrumentations SI; 745 SI.registerCallbacks(PIC); 746 747 if (LlvmSelfProfiler){ 748 LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback); 749 } 750 751 Optional<PGOOptions> PGOOpt; 752 if (PGOGenPath) { 753 assert(!PGOUsePath); 754 PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr); 755 } else if (PGOUsePath) { 756 assert(!PGOGenPath); 757 PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse); 758 } 759 760 PassBuilder PB(TM, PTO, PGOOpt, &PIC); 761 762 // FIXME: We may want to expose this as an option. 763 bool DebugPassManager = false; 764 LoopAnalysisManager LAM(DebugPassManager); 765 FunctionAnalysisManager FAM(DebugPassManager); 766 CGSCCAnalysisManager CGAM(DebugPassManager); 767 ModuleAnalysisManager MAM(DebugPassManager); 768 769 FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); 770 771 Triple TargetTriple(TheModule->getTargetTriple()); 772 std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple)); 773 if (DisableSimplifyLibCalls) 774 TLII->disableAllFunctions(); 775 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); 776 777 PB.registerModuleAnalyses(MAM); 778 PB.registerCGSCCAnalyses(CGAM); 779 PB.registerFunctionAnalyses(FAM); 780 PB.registerLoopAnalyses(LAM); 781 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); 782 783 // We manually collect pipeline callbacks so we can apply them at O0, where the 784 // PassBuilder does not create a pipeline. 785 std::vector<std::function<void(ModulePassManager &)>> PipelineStartEPCallbacks; 786 std::vector<std::function<void(FunctionPassManager &, PassBuilder::OptimizationLevel)>> 787 OptimizerLastEPCallbacks; 788 789 if (VerifyIR) { 790 PipelineStartEPCallbacks.push_back([VerifyIR](ModulePassManager &MPM) { 791 MPM.addPass(VerifierPass()); 792 }); 793 } 794 795 if (SanitizerOptions) { 796 if (SanitizerOptions->SanitizeMemory) { 797 MemorySanitizerOptions Options( 798 SanitizerOptions->SanitizeMemoryTrackOrigins, 799 SanitizerOptions->SanitizeRecover, 800 /*CompileKernel=*/false); 801#if LLVM_VERSION_GE(10, 0) 802 PipelineStartEPCallbacks.push_back([Options](ModulePassManager &MPM) { 803 MPM.addPass(MemorySanitizerPass(Options)); 804 }); 805#endif 806 OptimizerLastEPCallbacks.push_back( 807 [Options](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { 808 FPM.addPass(MemorySanitizerPass(Options)); 809 } 810 ); 811 } 812 813 if (SanitizerOptions->SanitizeThread) { 814#if LLVM_VERSION_GE(10, 0) 815 PipelineStartEPCallbacks.push_back([](ModulePassManager &MPM) { 816 MPM.addPass(ThreadSanitizerPass()); 817 }); 818#endif 819 OptimizerLastEPCallbacks.push_back( 820 [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { 821 FPM.addPass(ThreadSanitizerPass()); 822 } 823 ); 824 } 825 826 if (SanitizerOptions->SanitizeAddress) { 827 PipelineStartEPCallbacks.push_back([&](ModulePassManager &MPM) { 828 MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>()); 829 }); 830 OptimizerLastEPCallbacks.push_back( 831 [SanitizerOptions](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) { 832 FPM.addPass(AddressSanitizerPass( 833 /*CompileKernel=*/false, SanitizerOptions->SanitizeRecover, 834 /*UseAfterScope=*/true)); 835 } 836 ); 837 PipelineStartEPCallbacks.push_back( 838 [SanitizerOptions](ModulePassManager &MPM) { 839 MPM.addPass(ModuleAddressSanitizerPass( 840 /*CompileKernel=*/false, SanitizerOptions->SanitizeRecover)); 841 } 842 ); 843 } 844 } 845 846 ModulePassManager MPM(DebugPassManager); 847 if (!NoPrepopulatePasses) { 848 if (OptLevel == PassBuilder::O0) { 849 for (const auto &C : PipelineStartEPCallbacks) 850 C(MPM); 851 852 if (!OptimizerLastEPCallbacks.empty()) { 853 FunctionPassManager FPM(DebugPassManager); 854 for (const auto &C : OptimizerLastEPCallbacks) 855 C(FPM, OptLevel); 856 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 857 } 858 859 MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/false)); 860 861#if LLVM_VERSION_GE(10, 0) 862 if (PGOOpt) { 863 PB.addPGOInstrPassesForO0( 864 MPM, DebugPassManager, PGOOpt->Action == PGOOptions::IRInstr, 865 /*IsCS=*/false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile); 866 } 867#endif 868 } else { 869 for (const auto &C : PipelineStartEPCallbacks) 870 PB.registerPipelineStartEPCallback(C); 871 if (OptStage != LLVMRustOptStage::PreLinkThinLTO) { 872 for (const auto &C : OptimizerLastEPCallbacks) 873 PB.registerOptimizerLastEPCallback(C); 874 } 875 876 switch (OptStage) { 877 case LLVMRustOptStage::PreLinkNoLTO: 878 MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager); 879 break; 880 case LLVMRustOptStage::PreLinkThinLTO: 881 MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager); 882 if (!OptimizerLastEPCallbacks.empty()) { 883 FunctionPassManager FPM(DebugPassManager); 884 for (const auto &C : OptimizerLastEPCallbacks) 885 C(FPM, OptLevel); 886 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 887 } 888 break; 889 case LLVMRustOptStage::PreLinkFatLTO: 890 MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager); 891 break; 892 case LLVMRustOptStage::ThinLTO: 893 // FIXME: Does it make sense to pass the ModuleSummaryIndex? 894 // It only seems to be needed for C++ specific optimizations. 895 MPM = PB.buildThinLTODefaultPipeline(OptLevel, DebugPassManager, nullptr); 896 break; 897 case LLVMRustOptStage::FatLTO: 898 MPM = PB.buildLTODefaultPipeline(OptLevel, DebugPassManager, nullptr); 899 break; 900 } 901 } 902 } 903 904 if (UseThinLTOBuffers) { 905 MPM.addPass(CanonicalizeAliasesPass()); 906 MPM.addPass(NameAnonGlobalPass()); 907 } 908 909 // Upgrade all calls to old intrinsics first. 910 for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;) 911 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove 912 913 MPM.run(*TheModule, MAM); 914#else 915 // The new pass manager has been available for a long time, 916 // but we don't bother supporting it on old LLVM versions. 917 report_fatal_error("New pass manager only supported since LLVM 9"); 918#endif 919} 920 921// Callback to demangle function name 922// Parameters: 923// * name to be demangled 924// * name len 925// * output buffer 926// * output buffer len 927// Returns len of demangled string, or 0 if demangle failed. 928typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t); 929 930 931namespace { 932 933class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter { 934 DemangleFn Demangle; 935 std::vector<char> Buf; 936 937public: 938 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {} 939 940 // Return empty string if demangle failed 941 // or if name does not need to be demangled 942 StringRef CallDemangle(StringRef name) { 943 if (!Demangle) { 944 return StringRef(); 945 } 946 947 if (Buf.size() < name.size() * 2) { 948 // Semangled name usually shorter than mangled, 949 // but allocate twice as much memory just in case 950 Buf.resize(name.size() * 2); 951 } 952 953 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size()); 954 if (!R) { 955 // Demangle failed. 956 return StringRef(); 957 } 958 959 auto Demangled = StringRef(Buf.data(), R); 960 if (Demangled == name) { 961 // Do not print anything if demangled name is equal to mangled. 962 return StringRef(); 963 } 964 965 return Demangled; 966 } 967 968 void emitFunctionAnnot(const Function *F, 969 formatted_raw_ostream &OS) override { 970 StringRef Demangled = CallDemangle(F->getName()); 971 if (Demangled.empty()) { 972 return; 973 } 974 975 OS << "; " << Demangled << "\n"; 976 } 977 978 void emitInstructionAnnot(const Instruction *I, 979 formatted_raw_ostream &OS) override { 980 const char *Name; 981 const Value *Value; 982 if (const CallInst *CI = dyn_cast<CallInst>(I)) { 983 Name = "call"; 984 Value = CI->getCalledValue(); 985 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) { 986 Name = "invoke"; 987 Value = II->getCalledValue(); 988 } else { 989 // Could demangle more operations, e. g. 990 // `store %place, @function`. 991 return; 992 } 993 994 if (!Value->hasName()) { 995 return; 996 } 997 998 StringRef Demangled = CallDemangle(Value->getName()); 999 if (Demangled.empty()) { 1000 return; 1001 } 1002 1003 OS << "; " << Name << " " << Demangled << "\n"; 1004 } 1005}; 1006 1007} // namespace 1008 1009extern "C" LLVMRustResult 1010LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) { 1011 std::string ErrorInfo; 1012 std::error_code EC; 1013 raw_fd_ostream OS(Path, EC, sys::fs::F_None); 1014 if (EC) 1015 ErrorInfo = EC.message(); 1016 if (ErrorInfo != "") { 1017 LLVMRustSetLastError(ErrorInfo.c_str()); 1018 return LLVMRustResult::Failure; 1019 } 1020 1021 RustAssemblyAnnotationWriter AAW(Demangle); 1022 formatted_raw_ostream FOS(OS); 1023 unwrap(M)->print(FOS, &AAW); 1024 1025 return LLVMRustResult::Success; 1026} 1027 1028extern "C" void LLVMRustPrintPasses() { 1029 LLVMInitializePasses(); 1030 struct MyListener : PassRegistrationListener { 1031 void passEnumerate(const PassInfo *Info) { 1032 StringRef PassArg = Info->getPassArgument(); 1033 StringRef PassName = Info->getPassName(); 1034 if (!PassArg.empty()) { 1035 // These unsigned->signed casts could theoretically overflow, but 1036 // realistically never will (and even if, the result is implementation 1037 // defined rather plain UB). 1038 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(), 1039 (int)PassName.size(), PassName.data()); 1040 } 1041 } 1042 } Listener; 1043 1044 PassRegistry *PR = PassRegistry::getPassRegistry(); 1045 PR->enumerateWith(&Listener); 1046} 1047 1048extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR, 1049 bool AddLifetimes) { 1050 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes); 1051} 1052 1053extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, 1054 size_t Len) { 1055 llvm::legacy::PassManager passes; 1056 1057 auto PreserveFunctions = [=](const GlobalValue &GV) { 1058 for (size_t I = 0; I < Len; I++) { 1059 if (GV.getName() == Symbols[I]) { 1060 return true; 1061 } 1062 } 1063 return false; 1064 }; 1065 1066 passes.add(llvm::createInternalizePass(PreserveFunctions)); 1067 1068 passes.run(*unwrap(M)); 1069} 1070 1071extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) { 1072 for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E; 1073 ++GV) { 1074 GV->setDoesNotThrow(); 1075 Function *F = dyn_cast<Function>(GV); 1076 if (F == nullptr) 1077 continue; 1078 1079 for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) { 1080 for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) { 1081 if (isa<InvokeInst>(I)) { 1082 InvokeInst *CI = cast<InvokeInst>(I); 1083 CI->setDoesNotThrow(); 1084 } 1085 } 1086 } 1087 } 1088} 1089 1090extern "C" void 1091LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module, 1092 LLVMTargetMachineRef TMR) { 1093 TargetMachine *Target = unwrap(TMR); 1094 unwrap(Module)->setDataLayout(Target->createDataLayout()); 1095} 1096 1097extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) { 1098 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC); 1099} 1100 1101extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { 1102 unwrap(M)->setPIELevel(PIELevel::Level::Large); 1103} 1104 1105// Here you'll find an implementation of ThinLTO as used by the Rust compiler 1106// right now. This ThinLTO support is only enabled on "recent ish" versions of 1107// LLVM, and otherwise it's just blanket rejected from other compilers. 1108// 1109// Most of this implementation is straight copied from LLVM. At the time of 1110// this writing it wasn't *quite* suitable to reuse more code from upstream 1111// for our purposes, but we should strive to upstream this support once it's 1112// ready to go! I figure we may want a bit of testing locally first before 1113// sending this upstream to LLVM. I hear though they're quite eager to receive 1114// feedback like this! 1115// 1116// If you're reading this code and wondering "what in the world" or you're 1117// working "good lord by LLVM upgrade is *still* failing due to these bindings" 1118// then fear not! (ok maybe fear a little). All code here is mostly based 1119// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM. 1120// 1121// You'll find that the general layout here roughly corresponds to the `run` 1122// method in that file as well as `ProcessThinLTOModule`. Functions are 1123// specifically commented below as well, but if you're updating this code 1124// or otherwise trying to understand it, the LLVM source will be useful in 1125// interpreting the mysteries within. 1126// 1127// Otherwise I'll apologize in advance, it probably requires a relatively 1128// significant investment on your part to "truly understand" what's going on 1129// here. Not saying I do myself, but it took me awhile staring at LLVM's source 1130// and various online resources about ThinLTO to make heads or tails of all 1131// this. 1132 1133// This is a shared data structure which *must* be threadsafe to share 1134// read-only amongst threads. This also corresponds basically to the arguments 1135// of the `ProcessThinLTOModule` function in the LLVM source. 1136struct LLVMRustThinLTOData { 1137 // The combined index that is the global analysis over all modules we're 1138 // performing ThinLTO for. This is mostly managed by LLVM. 1139 ModuleSummaryIndex Index; 1140 1141 // All modules we may look at, stored as in-memory serialized versions. This 1142 // is later used when inlining to ensure we can extract any module to inline 1143 // from. 1144 StringMap<MemoryBufferRef> ModuleMap; 1145 1146 // A set that we manage of everything we *don't* want internalized. Note that 1147 // this includes all transitive references right now as well, but it may not 1148 // always! 1149 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols; 1150 1151 // Not 100% sure what these are, but they impact what's internalized and 1152 // what's inlined across modules, I believe. 1153 StringMap<FunctionImporter::ImportMapTy> ImportLists; 1154 StringMap<FunctionImporter::ExportSetTy> ExportLists; 1155 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries; 1156 1157 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} 1158}; 1159 1160// Just an argument to the `LLVMRustCreateThinLTOData` function below. 1161struct LLVMRustThinLTOModule { 1162 const char *identifier; 1163 const char *data; 1164 size_t len; 1165}; 1166 1167// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it 1168// does. 1169static const GlobalValueSummary * 1170getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { 1171 auto StrongDefForLinker = llvm::find_if( 1172 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) { 1173 auto Linkage = Summary->linkage(); 1174 return !GlobalValue::isAvailableExternallyLinkage(Linkage) && 1175 !GlobalValue::isWeakForLinker(Linkage); 1176 }); 1177 if (StrongDefForLinker != GVSummaryList.end()) 1178 return StrongDefForLinker->get(); 1179 1180 auto FirstDefForLinker = llvm::find_if( 1181 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) { 1182 auto Linkage = Summary->linkage(); 1183 return !GlobalValue::isAvailableExternallyLinkage(Linkage); 1184 }); 1185 if (FirstDefForLinker == GVSummaryList.end()) 1186 return nullptr; 1187 return FirstDefForLinker->get(); 1188} 1189 1190// The main entry point for creating the global ThinLTO analysis. The structure 1191// here is basically the same as before threads are spawned in the `run` 1192// function of `lib/LTO/ThinLTOCodeGenerator.cpp`. 1193extern "C" LLVMRustThinLTOData* 1194LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, 1195 int num_modules, 1196 const char **preserved_symbols, 1197 int num_symbols) { 1198#if LLVM_VERSION_GE(10, 0) 1199 auto Ret = std::make_unique<LLVMRustThinLTOData>(); 1200#else 1201 auto Ret = llvm::make_unique<LLVMRustThinLTOData>(); 1202#endif 1203 1204 // Load each module's summary and merge it into one combined index 1205 for (int i = 0; i < num_modules; i++) { 1206 auto module = &modules[i]; 1207 StringRef buffer(module->data, module->len); 1208 MemoryBufferRef mem_buffer(buffer, module->identifier); 1209 1210 Ret->ModuleMap[module->identifier] = mem_buffer; 1211 1212 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { 1213 LLVMRustSetLastError(toString(std::move(Err)).c_str()); 1214 return nullptr; 1215 } 1216 } 1217 1218 // Collect for each module the list of function it defines (GUID -> Summary) 1219 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries); 1220 1221 // Convert the preserved symbols set from string to GUID, this is then needed 1222 // for internalization. 1223 for (int i = 0; i < num_symbols; i++) { 1224 auto GUID = GlobalValue::getGUID(preserved_symbols[i]); 1225 Ret->GUIDPreservedSymbols.insert(GUID); 1226 } 1227 1228 // Collect the import/export lists for all modules from the call-graph in the 1229 // combined index 1230 // 1231 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` 1232 auto deadIsPrevailing = [&](GlobalValue::GUID G) { 1233 return PrevailingType::Unknown; 1234 }; 1235#if LLVM_VERSION_GE(8, 0) 1236 // We don't have a complete picture in our use of ThinLTO, just our immediate 1237 // crate, so we need `ImportEnabled = false` to limit internalization. 1238 // Otherwise, we sometimes lose `static` values -- see #60184. 1239 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, 1240 deadIsPrevailing, /* ImportEnabled = */ false); 1241#else 1242 computeDeadSymbols(Ret->Index, Ret->GUIDPreservedSymbols, deadIsPrevailing); 1243#endif 1244 ComputeCrossModuleImport( 1245 Ret->Index, 1246 Ret->ModuleToDefinedGVSummaries, 1247 Ret->ImportLists, 1248 Ret->ExportLists 1249 ); 1250 1251 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it 1252 // impacts the caching. 1253 // 1254 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this 1255 // being lifted from `lib/LTO/LTO.cpp` as well 1256 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; 1257 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; 1258 for (auto &I : Ret->Index) { 1259 if (I.second.SummaryList.size() > 1) 1260 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList); 1261 } 1262 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { 1263 const auto &Prevailing = PrevailingCopy.find(GUID); 1264 if (Prevailing == PrevailingCopy.end()) 1265 return true; 1266 return Prevailing->second == S; 1267 }; 1268 auto recordNewLinkage = [&](StringRef ModuleIdentifier, 1269 GlobalValue::GUID GUID, 1270 GlobalValue::LinkageTypes NewLinkage) { 1271 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; 1272 }; 1273#if LLVM_VERSION_GE(9, 0) 1274 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage, 1275 Ret->GUIDPreservedSymbols); 1276#elif LLVM_VERSION_GE(8, 0) 1277 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage); 1278#else 1279 thinLTOResolveWeakForLinkerInIndex(Ret->Index, isPrevailing, recordNewLinkage); 1280#endif 1281 1282 // Here we calculate an `ExportedGUIDs` set for use in the `isExported` 1283 // callback below. This callback below will dictate the linkage for all 1284 // summaries in the index, and we basically just only want to ensure that dead 1285 // symbols are internalized. Otherwise everything that's already external 1286 // linkage will stay as external, and internal will stay as internal. 1287 std::set<GlobalValue::GUID> ExportedGUIDs; 1288 for (auto &List : Ret->Index) { 1289 for (auto &GVS: List.second.SummaryList) { 1290 if (GlobalValue::isLocalLinkage(GVS->linkage())) 1291 continue; 1292 auto GUID = GVS->getOriginalName(); 1293 if (GVS->flags().Live) 1294 ExportedGUIDs.insert(GUID); 1295 } 1296 } 1297#if LLVM_VERSION_GE(10, 0) 1298 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) { 1299 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); 1300 return (ExportList != Ret->ExportLists.end() && 1301 ExportList->second.count(VI)) || 1302 ExportedGUIDs.count(VI.getGUID()); 1303 }; 1304 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing); 1305#else 1306 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) { 1307 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); 1308 return (ExportList != Ret->ExportLists.end() && 1309 ExportList->second.count(GUID)) || 1310 ExportedGUIDs.count(GUID); 1311 }; 1312 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported); 1313#endif 1314 1315 return Ret.release(); 1316} 1317 1318extern "C" void 1319LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { 1320 delete Data; 1321} 1322 1323// Below are the various passes that happen *per module* when doing ThinLTO. 1324// 1325// In other words, these are the functions that are all run concurrently 1326// with one another, one per module. The passes here correspond to the analysis 1327// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the 1328// `ProcessThinLTOModule` function. Here they're split up into separate steps 1329// so rustc can save off the intermediate bytecode between each step. 1330 1331extern "C" bool 1332LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 1333 Module &Mod = *unwrap(M); 1334 if (renameModuleForThinLTO(Mod, Data->Index)) { 1335 LLVMRustSetLastError("renameModuleForThinLTO failed"); 1336 return false; 1337 } 1338 return true; 1339} 1340 1341extern "C" bool 1342LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 1343 Module &Mod = *unwrap(M); 1344 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); 1345#if LLVM_VERSION_GE(8, 0) 1346 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals); 1347#else 1348 thinLTOResolveWeakForLinkerModule(Mod, DefinedGlobals); 1349#endif 1350 return true; 1351} 1352 1353extern "C" bool 1354LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 1355 Module &Mod = *unwrap(M); 1356 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); 1357 thinLTOInternalizeModule(Mod, DefinedGlobals); 1358 return true; 1359} 1360 1361extern "C" bool 1362LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 1363 Module &Mod = *unwrap(M); 1364 1365 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier()); 1366 auto Loader = [&](StringRef Identifier) { 1367 const auto &Memory = Data->ModuleMap.lookup(Identifier); 1368 auto &Context = Mod.getContext(); 1369 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true); 1370 1371 if (!MOrErr) 1372 return MOrErr; 1373 1374 // The rest of this closure is a workaround for 1375 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports 1376 // we accidentally import wasm custom sections into different modules, 1377 // duplicating them by in the final output artifact. 1378 // 1379 // The issue is worked around here by manually removing the 1380 // `wasm.custom_sections` named metadata node from any imported module. This 1381 // we know isn't used by any optimization pass so there's no need for it to 1382 // be imported. 1383 // 1384 // Note that the metadata is currently lazily loaded, so we materialize it 1385 // here before looking up if there's metadata inside. The `FunctionImporter` 1386 // will immediately materialize metadata anyway after an import, so this 1387 // shouldn't be a perf hit. 1388 if (Error Err = (*MOrErr)->materializeMetadata()) { 1389 Expected<std::unique_ptr<Module>> Ret(std::move(Err)); 1390 return Ret; 1391 } 1392 1393 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections"); 1394 if (WasmCustomSections) 1395 WasmCustomSections->eraseFromParent(); 1396 1397 return MOrErr; 1398 }; 1399 FunctionImporter Importer(Data->Index, Loader); 1400 Expected<bool> Result = Importer.importFunctions(Mod, ImportList); 1401 if (!Result) { 1402 LLVMRustSetLastError(toString(Result.takeError()).c_str()); 1403 return false; 1404 } 1405 return true; 1406} 1407 1408extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload 1409 const char*, // importing module name 1410 const char*); // imported module name 1411 1412// Calls `module_name_callback` for each module import done by ThinLTO. 1413// The callback is provided with regular null-terminated C strings. 1414extern "C" void 1415LLVMRustGetThinLTOModuleImports(const LLVMRustThinLTOData *data, 1416 LLVMRustModuleNameCallback module_name_callback, 1417 void* callback_payload) { 1418 for (const auto& importing_module : data->ImportLists) { 1419 const std::string importing_module_id = importing_module.getKey().str(); 1420 const auto& imports = importing_module.getValue(); 1421 for (const auto& imported_module : imports) { 1422 const std::string imported_module_id = imported_module.getKey().str(); 1423 module_name_callback(callback_payload, 1424 importing_module_id.c_str(), 1425 imported_module_id.c_str()); 1426 } 1427 } 1428} 1429 1430// This struct and various functions are sort of a hack right now, but the 1431// problem is that we've got in-memory LLVM modules after we generate and 1432// optimize all codegen-units for one compilation in rustc. To be compatible 1433// with the LTO support above we need to serialize the modules plus their 1434// ThinLTO summary into memory. 1435// 1436// This structure is basically an owned version of a serialize module, with 1437// a ThinLTO summary attached. 1438struct LLVMRustThinLTOBuffer { 1439 std::string data; 1440}; 1441 1442extern "C" LLVMRustThinLTOBuffer* 1443LLVMRustThinLTOBufferCreate(LLVMModuleRef M) { 1444#if LLVM_VERSION_GE(10, 0) 1445 auto Ret = std::make_unique<LLVMRustThinLTOBuffer>(); 1446#else 1447 auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>(); 1448#endif 1449 { 1450 raw_string_ostream OS(Ret->data); 1451 { 1452 legacy::PassManager PM; 1453 PM.add(createWriteThinLTOBitcodePass(OS)); 1454 PM.run(*unwrap(M)); 1455 } 1456 } 1457 return Ret.release(); 1458} 1459 1460extern "C" void 1461LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { 1462 delete Buffer; 1463} 1464 1465extern "C" const void* 1466LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { 1467 return Buffer->data.data(); 1468} 1469 1470extern "C" size_t 1471LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { 1472 return Buffer->data.length(); 1473} 1474 1475// This is what we used to parse upstream bitcode for actual ThinLTO 1476// processing. We'll call this once per module optimized through ThinLTO, and 1477// it'll be called concurrently on many threads. 1478extern "C" LLVMModuleRef 1479LLVMRustParseBitcodeForLTO(LLVMContextRef Context, 1480 const char *data, 1481 size_t len, 1482 const char *identifier) { 1483 StringRef Data(data, len); 1484 MemoryBufferRef Buffer(Data, identifier); 1485 unwrap(Context)->enableDebugTypeODRUniquing(); 1486 Expected<std::unique_ptr<Module>> SrcOrError = 1487 parseBitcodeFile(Buffer, *unwrap(Context)); 1488 if (!SrcOrError) { 1489 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); 1490 return nullptr; 1491 } 1492 return wrap(std::move(*SrcOrError).release()); 1493} 1494 1495// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See 1496// the comment in `back/lto.rs` for why this exists. 1497extern "C" void 1498LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod, 1499 DICompileUnit **A, 1500 DICompileUnit **B) { 1501 Module *M = unwrap(Mod); 1502 DICompileUnit **Cur = A; 1503 DICompileUnit **Next = B; 1504 for (DICompileUnit *CU : M->debug_compile_units()) { 1505 *Cur = CU; 1506 Cur = Next; 1507 Next = nullptr; 1508 if (Cur == nullptr) 1509 break; 1510 } 1511} 1512 1513// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See 1514// the comment in `back/lto.rs` for why this exists. 1515extern "C" void 1516LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) { 1517 Module *M = unwrap(Mod); 1518 1519 // If the original source module didn't have a `DICompileUnit` then try to 1520 // merge all the existing compile units. If there aren't actually any though 1521 // then there's not much for us to do so return. 1522 if (Unit == nullptr) { 1523 for (DICompileUnit *CU : M->debug_compile_units()) { 1524 Unit = CU; 1525 break; 1526 } 1527 if (Unit == nullptr) 1528 return; 1529 } 1530 1531 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and 1532 // process it recursively. Note that we specifically iterate over instructions 1533 // to ensure we feed everything into it. 1534 DebugInfoFinder Finder; 1535 Finder.processModule(*M); 1536 for (Function &F : M->functions()) { 1537 for (auto &FI : F) { 1538 for (Instruction &BI : FI) { 1539 if (auto Loc = BI.getDebugLoc()) 1540 Finder.processLocation(*M, Loc); 1541 if (auto DVI = dyn_cast<DbgValueInst>(&BI)) 1542 Finder.processValue(*M, DVI); 1543 if (auto DDI = dyn_cast<DbgDeclareInst>(&BI)) 1544 Finder.processDeclare(*M, DDI); 1545 } 1546 } 1547 } 1548 1549 // After we've found all our debuginfo, rewrite all subprograms to point to 1550 // the same `DICompileUnit`. 1551 for (auto &F : Finder.subprograms()) { 1552 F->replaceUnit(Unit); 1553 } 1554 1555 // Erase any other references to other `DICompileUnit` instances, the verifier 1556 // will later ensure that we don't actually have any other stale references to 1557 // worry about. 1558 auto *MD = M->getNamedMetadata("llvm.dbg.cu"); 1559 MD->clearOperands(); 1560 MD->addOperand(Unit); 1561}