fork/rust
git clone https://git.julia.blue/fork/rust
5e1a799842b
1.41.0
1#include <stdio.h> 2 3#include <vector> 4#include <set> 5 6#include "rustllvm.h" 7 8#include "llvm/Analysis/TargetLibraryInfo.h" 9#include "llvm/Analysis/TargetTransformInfo.h" 10#include "llvm/CodeGen/TargetSubtargetInfo.h" 11#include "llvm/IR/AutoUpgrade.h" 12#include "llvm/IR/AssemblyAnnotationWriter.h" 13#include "llvm/IR/IntrinsicInst.h" 14#include "llvm/Support/CBindingWrapping.h" 15#include "llvm/Support/FileSystem.h" 16#include "llvm/Support/Host.h" 17#include "llvm/Target/TargetMachine.h" 18#include "llvm/Transforms/IPO/PassManagerBuilder.h" 19#include "llvm/Transforms/IPO/AlwaysInliner.h" 20#include "llvm/Transforms/IPO/FunctionImport.h" 21#include "llvm/Transforms/Utils/FunctionImportUtils.h" 22#include "llvm/LTO/LTO.h" 23#include "llvm-c/Transforms/PassManagerBuilder.h" 24 25#include "llvm/Transforms/Instrumentation.h" 26#if LLVM_VERSION_GE(9, 0) 27#include "llvm/Transforms/Instrumentation/AddressSanitizer.h" 28#endif 29#if LLVM_VERSION_GE(8, 0) 30#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" 31#include "llvm/Transforms/Instrumentation/MemorySanitizer.h" 32#endif 33 34using namespace llvm; 35using namespace llvm::legacy; 36 37typedef struct LLVMOpaquePass *LLVMPassRef; 38typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; 39 40DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef) 41DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) 42DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder, 43 LLVMPassManagerBuilderRef) 44 45extern "C" void LLVMInitializePasses() { 46 PassRegistry &Registry = *PassRegistry::getPassRegistry(); 47 initializeCore(Registry); 48 initializeCodeGen(Registry); 49 initializeScalarOpts(Registry); 50 initializeVectorization(Registry); 51 initializeIPO(Registry); 52 initializeAnalysis(Registry); 53 initializeTransformUtils(Registry); 54 initializeInstCombine(Registry); 55 initializeInstrumentation(Registry); 56 initializeTarget(Registry); 57} 58 59enum class LLVMRustPassKind { 60 Other, 61 Function, 62 Module, 63}; 64 65static LLVMRustPassKind toRust(PassKind Kind) { 66 switch (Kind) { 67 case PT_Function: 68 return LLVMRustPassKind::Function; 69 case PT_Module: 70 return LLVMRustPassKind::Module; 71 default: 72 return LLVMRustPassKind::Other; 73 } 74} 75 76extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) { 77 StringRef SR(PassName); 78 PassRegistry *PR = PassRegistry::getPassRegistry(); 79 80 const PassInfo *PI = PR->getPassInfo(SR); 81 if (PI) { 82 return wrap(PI->createPass()); 83 } 84 return nullptr; 85} 86 87extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) { 88 const bool CompileKernel = false; 89 90 return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover)); 91} 92 93extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) { 94 const bool CompileKernel = false; 95 96#if LLVM_VERSION_GE(9, 0) 97 return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover)); 98#else 99 return wrap(createAddressSanitizerModulePass(CompileKernel, Recover)); 100#endif 101} 102 103extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) { 104#if LLVM_VERSION_GE(9, 0) 105 const bool CompileKernel = false; 106 107 return wrap(createMemorySanitizerLegacyPassPass( 108 MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel})); 109#elif LLVM_VERSION_GE(8, 0) 110 return wrap(createMemorySanitizerLegacyPassPass(TrackOrigins, Recover)); 111#else 112 return wrap(createMemorySanitizerPass(TrackOrigins, Recover)); 113#endif 114} 115 116extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() { 117#if LLVM_VERSION_GE(8, 0) 118 return wrap(createThreadSanitizerLegacyPassPass()); 119#else 120 return wrap(createThreadSanitizerPass()); 121#endif 122} 123 124extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) { 125 assert(RustPass); 126 Pass *Pass = unwrap(RustPass); 127 return toRust(Pass->getPassKind()); 128} 129 130extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) { 131 assert(RustPass); 132 Pass *Pass = unwrap(RustPass); 133 PassManagerBase *PMB = unwrap(PMR); 134 PMB->add(Pass); 135} 136 137extern "C" 138void LLVMRustPassManagerBuilderPopulateThinLTOPassManager( 139 LLVMPassManagerBuilderRef PMBR, 140 LLVMPassManagerRef PMR 141) { 142 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR)); 143} 144 145extern "C" 146void LLVMRustAddLastExtensionPasses( 147 LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) { 148 auto AddExtensionPasses = [Passes, NumPasses]( 149 const PassManagerBuilder &Builder, PassManagerBase &PM) { 150 for (size_t I = 0; I < NumPasses; I++) { 151 PM.add(unwrap(Passes[I])); 152 } 153 }; 154 // Add the passes to both of the pre-finalization extension points, 155 // so they are run for optimized and non-optimized builds. 156 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast, 157 AddExtensionPasses); 158 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, 159 AddExtensionPasses); 160} 161 162#ifdef LLVM_COMPONENT_X86 163#define SUBTARGET_X86 SUBTARGET(X86) 164#else 165#define SUBTARGET_X86 166#endif 167 168#ifdef LLVM_COMPONENT_ARM 169#define SUBTARGET_ARM SUBTARGET(ARM) 170#else 171#define SUBTARGET_ARM 172#endif 173 174#ifdef LLVM_COMPONENT_AARCH64 175#define SUBTARGET_AARCH64 SUBTARGET(AArch64) 176#else 177#define SUBTARGET_AARCH64 178#endif 179 180#ifdef LLVM_COMPONENT_MIPS 181#define SUBTARGET_MIPS SUBTARGET(Mips) 182#else 183#define SUBTARGET_MIPS 184#endif 185 186#ifdef LLVM_COMPONENT_POWERPC 187#define SUBTARGET_PPC SUBTARGET(PPC) 188#else 189#define SUBTARGET_PPC 190#endif 191 192#ifdef LLVM_COMPONENT_SYSTEMZ 193#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ) 194#else 195#define SUBTARGET_SYSTEMZ 196#endif 197 198#ifdef LLVM_COMPONENT_MSP430 199#define SUBTARGET_MSP430 SUBTARGET(MSP430) 200#else 201#define SUBTARGET_MSP430 202#endif 203 204#ifdef LLVM_COMPONENT_RISCV 205#define SUBTARGET_RISCV SUBTARGET(RISCV) 206#else 207#define SUBTARGET_RISCV 208#endif 209 210#ifdef LLVM_COMPONENT_SPARC 211#define SUBTARGET_SPARC SUBTARGET(Sparc) 212#else 213#define SUBTARGET_SPARC 214#endif 215 216#ifdef LLVM_COMPONENT_HEXAGON 217#define SUBTARGET_HEXAGON SUBTARGET(Hexagon) 218#else 219#define SUBTARGET_HEXAGON 220#endif 221 222#define GEN_SUBTARGETS \ 223 SUBTARGET_X86 \ 224 SUBTARGET_ARM \ 225 SUBTARGET_AARCH64 \ 226 SUBTARGET_MIPS \ 227 SUBTARGET_PPC \ 228 SUBTARGET_SYSTEMZ \ 229 SUBTARGET_MSP430 \ 230 SUBTARGET_SPARC \ 231 SUBTARGET_HEXAGON \ 232 SUBTARGET_RISCV \ 233 234#define SUBTARGET(x) \ 235 namespace llvm { \ 236 extern const SubtargetFeatureKV x##FeatureKV[]; \ 237 extern const SubtargetFeatureKV x##SubTypeKV[]; \ 238 } 239 240GEN_SUBTARGETS 241#undef SUBTARGET 242 243extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, 244 const char *Feature) { 245 TargetMachine *Target = unwrap(TM); 246 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 247 return MCInfo->checkFeatures(std::string("+") + Feature); 248} 249 250enum class LLVMRustCodeModel { 251 Other, 252 Small, 253 Kernel, 254 Medium, 255 Large, 256 None, 257}; 258 259static CodeModel::Model fromRust(LLVMRustCodeModel Model) { 260 switch (Model) { 261 case LLVMRustCodeModel::Small: 262 return CodeModel::Small; 263 case LLVMRustCodeModel::Kernel: 264 return CodeModel::Kernel; 265 case LLVMRustCodeModel::Medium: 266 return CodeModel::Medium; 267 case LLVMRustCodeModel::Large: 268 return CodeModel::Large; 269 default: 270 report_fatal_error("Bad CodeModel."); 271 } 272} 273 274enum class LLVMRustCodeGenOptLevel { 275 Other, 276 None, 277 Less, 278 Default, 279 Aggressive, 280}; 281 282static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) { 283 switch (Level) { 284 case LLVMRustCodeGenOptLevel::None: 285 return CodeGenOpt::None; 286 case LLVMRustCodeGenOptLevel::Less: 287 return CodeGenOpt::Less; 288 case LLVMRustCodeGenOptLevel::Default: 289 return CodeGenOpt::Default; 290 case LLVMRustCodeGenOptLevel::Aggressive: 291 return CodeGenOpt::Aggressive; 292 default: 293 report_fatal_error("Bad CodeGenOptLevel."); 294 } 295} 296 297enum class LLVMRustRelocMode { 298 Default, 299 Static, 300 PIC, 301 DynamicNoPic, 302 ROPI, 303 RWPI, 304 ROPIRWPI, 305}; 306 307static Optional<Reloc::Model> fromRust(LLVMRustRelocMode RustReloc) { 308 switch (RustReloc) { 309 case LLVMRustRelocMode::Default: 310 return None; 311 case LLVMRustRelocMode::Static: 312 return Reloc::Static; 313 case LLVMRustRelocMode::PIC: 314 return Reloc::PIC_; 315 case LLVMRustRelocMode::DynamicNoPic: 316 return Reloc::DynamicNoPIC; 317 case LLVMRustRelocMode::ROPI: 318 return Reloc::ROPI; 319 case LLVMRustRelocMode::RWPI: 320 return Reloc::RWPI; 321 case LLVMRustRelocMode::ROPIRWPI: 322 return Reloc::ROPI_RWPI; 323 } 324 report_fatal_error("Bad RelocModel."); 325} 326 327#ifdef LLVM_RUSTLLVM 328/// getLongestEntryLength - Return the length of the longest entry in the table. 329template<typename KV> 330static size_t getLongestEntryLength(ArrayRef<KV> Table) { 331 size_t MaxLen = 0; 332 for (auto &I : Table) 333 MaxLen = std::max(MaxLen, std::strlen(I.Key)); 334 return MaxLen; 335} 336 337extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) { 338 const TargetMachine *Target = unwrap(TM); 339 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 340 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch(); 341 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); 342 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable(); 343 unsigned MaxCPULen = getLongestEntryLength(CPUTable); 344 345 printf("Available CPUs for this target:\n"); 346 if (HostArch == TargetArch) { 347 const StringRef HostCPU = sys::getHostCPUName(); 348 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n", 349 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data()); 350 } 351 for (auto &CPU : CPUTable) 352 printf(" %-*s\n", MaxCPULen, CPU.Key); 353 printf("\n"); 354} 355 356extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) { 357 const TargetMachine *Target = unwrap(TM); 358 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 359 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable(); 360 unsigned MaxFeatLen = getLongestEntryLength(FeatTable); 361 362 printf("Available features for this target:\n"); 363 for (auto &Feature : FeatTable) 364 printf(" %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc); 365 printf("\n"); 366 367 printf("Use +feature to enable a feature, or -feature to disable it.\n" 368 "For example, rustc -C -target-cpu=mycpu -C " 369 "target-feature=+feature1,-feature2\n\n"); 370} 371 372#else 373 374extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) { 375 printf("Target CPU help is not supported by this LLVM version.\n\n"); 376} 377 378extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) { 379 printf("Target features help is not supported by this LLVM version.\n\n"); 380} 381#endif 382 383extern "C" const char* LLVMRustGetHostCPUName(size_t *len) { 384 StringRef Name = sys::getHostCPUName(); 385 *len = Name.size(); 386 return Name.data(); 387} 388 389extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( 390 const char *TripleStr, const char *CPU, const char *Feature, 391 const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocMode RustReloc, 392 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, 393 bool PositionIndependentExecutable, bool FunctionSections, 394 bool DataSections, 395 bool TrapUnreachable, 396 bool Singlethread, 397 bool AsmComments, 398 bool EmitStackSizeSection, 399 bool RelaxELFRelocations) { 400 401 auto OptLevel = fromRust(RustOptLevel); 402 auto RM = fromRust(RustReloc); 403 404 std::string Error; 405 Triple Trip(Triple::normalize(TripleStr)); 406 const llvm::Target *TheTarget = 407 TargetRegistry::lookupTarget(Trip.getTriple(), Error); 408 if (TheTarget == nullptr) { 409 LLVMRustSetLastError(Error.c_str()); 410 return nullptr; 411 } 412 413 TargetOptions Options; 414 415 Options.FloatABIType = FloatABI::Default; 416 if (UseSoftFloat) { 417 Options.FloatABIType = FloatABI::Soft; 418 } 419 Options.DataSections = DataSections; 420 Options.FunctionSections = FunctionSections; 421 Options.MCOptions.AsmVerbose = AsmComments; 422 Options.MCOptions.PreserveAsmComments = AsmComments; 423 Options.MCOptions.ABIName = ABIStr; 424 Options.RelaxELFRelocations = RelaxELFRelocations; 425 426 if (TrapUnreachable) { 427 // Tell LLVM to codegen `unreachable` into an explicit trap instruction. 428 // This limits the extent of possible undefined behavior in some cases, as 429 // it prevents control flow from "falling through" into whatever code 430 // happens to be laid out next in memory. 431 Options.TrapUnreachable = true; 432 } 433 434 if (Singlethread) { 435 Options.ThreadModel = ThreadModel::Single; 436 } 437 438 Options.EmitStackSizeSection = EmitStackSizeSection; 439 440 Optional<CodeModel::Model> CM; 441 if (RustCM != LLVMRustCodeModel::None) 442 CM = fromRust(RustCM); 443 TargetMachine *TM = TheTarget->createTargetMachine( 444 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); 445 return wrap(TM); 446} 447 448extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { 449 delete unwrap(TM); 450} 451 452extern "C" void LLVMRustConfigurePassManagerBuilder( 453 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel, 454 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO, 455 const char* PGOGenPath, const char* PGOUsePath) { 456 unwrap(PMBR)->MergeFunctions = MergeFunctions; 457 unwrap(PMBR)->SLPVectorize = SLPVectorize; 458 unwrap(PMBR)->OptLevel = fromRust(OptLevel); 459 unwrap(PMBR)->LoopVectorize = LoopVectorize; 460 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO; 461 462 if (PGOGenPath) { 463 assert(!PGOUsePath); 464 unwrap(PMBR)->EnablePGOInstrGen = true; 465 unwrap(PMBR)->PGOInstrGen = PGOGenPath; 466 } 467 if (PGOUsePath) { 468 assert(!PGOGenPath); 469 unwrap(PMBR)->PGOInstrUse = PGOUsePath; 470 } 471} 472 473// Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo` 474// field of a PassManagerBuilder, we expose our own method of doing so. 475extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR, 476 LLVMModuleRef M, 477 bool DisableSimplifyLibCalls) { 478 Triple TargetTriple(unwrap(M)->getTargetTriple()); 479 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple); 480 if (DisableSimplifyLibCalls) 481 TLI->disableAllFunctions(); 482 unwrap(PMBR)->LibraryInfo = TLI; 483} 484 485// Unfortunately, the LLVM C API doesn't provide a way to create the 486// TargetLibraryInfo pass, so we use this method to do so. 487extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M, 488 bool DisableSimplifyLibCalls) { 489 Triple TargetTriple(unwrap(M)->getTargetTriple()); 490 TargetLibraryInfoImpl TLII(TargetTriple); 491 if (DisableSimplifyLibCalls) 492 TLII.disableAllFunctions(); 493 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII)); 494} 495 496// Unfortunately, the LLVM C API doesn't provide an easy way of iterating over 497// all the functions in a module, so we do that manually here. You'll find 498// similar code in clang's BackendUtil.cpp file. 499extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR, 500 LLVMModuleRef M) { 501 llvm::legacy::FunctionPassManager *P = 502 unwrap<llvm::legacy::FunctionPassManager>(PMR); 503 P->doInitialization(); 504 505 // Upgrade all calls to old intrinsics first. 506 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;) 507 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove 508 509 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E; 510 ++I) 511 if (!I->isDeclaration()) 512 P->run(*I); 513 514 P->doFinalization(); 515} 516 517extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) { 518 // Initializing the command-line options more than once is not allowed. So, 519 // check if they've already been initialized. (This could happen if we're 520 // being called from rustpkg, for example). If the arguments change, then 521 // that's just kinda unfortunate. 522 static bool Initialized = false; 523 if (Initialized) 524 return; 525 Initialized = true; 526 cl::ParseCommandLineOptions(Argc, Argv); 527} 528 529enum class LLVMRustFileType { 530 Other, 531 AssemblyFile, 532 ObjectFile, 533}; 534 535static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) { 536 switch (Type) { 537 case LLVMRustFileType::AssemblyFile: 538 return TargetMachine::CGFT_AssemblyFile; 539 case LLVMRustFileType::ObjectFile: 540 return TargetMachine::CGFT_ObjectFile; 541 default: 542 report_fatal_error("Bad FileType."); 543 } 544} 545 546extern "C" LLVMRustResult 547LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, 548 LLVMModuleRef M, const char *Path, 549 LLVMRustFileType RustFileType) { 550 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR); 551 auto FileType = fromRust(RustFileType); 552 553 std::string ErrorInfo; 554 std::error_code EC; 555 raw_fd_ostream OS(Path, EC, sys::fs::F_None); 556 if (EC) 557 ErrorInfo = EC.message(); 558 if (ErrorInfo != "") { 559 LLVMRustSetLastError(ErrorInfo.c_str()); 560 return LLVMRustResult::Failure; 561 } 562 563 buffer_ostream BOS(OS); 564 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false); 565 PM->run(*unwrap(M)); 566 567 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output 568 // stream (OS), so the only real safe place to delete this is here? Don't we 569 // wish this was written in Rust? 570 LLVMDisposePassManager(PMR); 571 return LLVMRustResult::Success; 572} 573 574 575// Callback to demangle function name 576// Parameters: 577// * name to be demangled 578// * name len 579// * output buffer 580// * output buffer len 581// Returns len of demangled string, or 0 if demangle failed. 582typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t); 583 584 585namespace { 586 587class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter { 588 DemangleFn Demangle; 589 std::vector<char> Buf; 590 591public: 592 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {} 593 594 // Return empty string if demangle failed 595 // or if name does not need to be demangled 596 StringRef CallDemangle(StringRef name) { 597 if (!Demangle) { 598 return StringRef(); 599 } 600 601 if (Buf.size() < name.size() * 2) { 602 // Semangled name usually shorter than mangled, 603 // but allocate twice as much memory just in case 604 Buf.resize(name.size() * 2); 605 } 606 607 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size()); 608 if (!R) { 609 // Demangle failed. 610 return StringRef(); 611 } 612 613 auto Demangled = StringRef(Buf.data(), R); 614 if (Demangled == name) { 615 // Do not print anything if demangled name is equal to mangled. 616 return StringRef(); 617 } 618 619 return Demangled; 620 } 621 622 void emitFunctionAnnot(const Function *F, 623 formatted_raw_ostream &OS) override { 624 StringRef Demangled = CallDemangle(F->getName()); 625 if (Demangled.empty()) { 626 return; 627 } 628 629 OS << "; " << Demangled << "\n"; 630 } 631 632 void emitInstructionAnnot(const Instruction *I, 633 formatted_raw_ostream &OS) override { 634 const char *Name; 635 const Value *Value; 636 if (const CallInst *CI = dyn_cast<CallInst>(I)) { 637 Name = "call"; 638 Value = CI->getCalledValue(); 639 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) { 640 Name = "invoke"; 641 Value = II->getCalledValue(); 642 } else { 643 // Could demangle more operations, e. g. 644 // `store %place, @function`. 645 return; 646 } 647 648 if (!Value->hasName()) { 649 return; 650 } 651 652 StringRef Demangled = CallDemangle(Value->getName()); 653 if (Demangled.empty()) { 654 return; 655 } 656 657 OS << "; " << Name << " " << Demangled << "\n"; 658 } 659}; 660 661} // namespace 662 663extern "C" LLVMRustResult 664LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) { 665 std::string ErrorInfo; 666 std::error_code EC; 667 raw_fd_ostream OS(Path, EC, sys::fs::F_None); 668 if (EC) 669 ErrorInfo = EC.message(); 670 if (ErrorInfo != "") { 671 LLVMRustSetLastError(ErrorInfo.c_str()); 672 return LLVMRustResult::Failure; 673 } 674 675 RustAssemblyAnnotationWriter AAW(Demangle); 676 formatted_raw_ostream FOS(OS); 677 unwrap(M)->print(FOS, &AAW); 678 679 return LLVMRustResult::Success; 680} 681 682extern "C" void LLVMRustPrintPasses() { 683 LLVMInitializePasses(); 684 struct MyListener : PassRegistrationListener { 685 void passEnumerate(const PassInfo *Info) { 686 StringRef PassArg = Info->getPassArgument(); 687 StringRef PassName = Info->getPassName(); 688 if (!PassArg.empty()) { 689 // These unsigned->signed casts could theoretically overflow, but 690 // realistically never will (and even if, the result is implementation 691 // defined rather plain UB). 692 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(), 693 (int)PassName.size(), PassName.data()); 694 } 695 } 696 } Listener; 697 698 PassRegistry *PR = PassRegistry::getPassRegistry(); 699 PR->enumerateWith(&Listener); 700} 701 702extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR, 703 bool AddLifetimes) { 704 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes); 705} 706 707extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, 708 size_t Len) { 709 llvm::legacy::PassManager passes; 710 711 auto PreserveFunctions = [=](const GlobalValue &GV) { 712 for (size_t I = 0; I < Len; I++) { 713 if (GV.getName() == Symbols[I]) { 714 return true; 715 } 716 } 717 return false; 718 }; 719 720 passes.add(llvm::createInternalizePass(PreserveFunctions)); 721 722 passes.run(*unwrap(M)); 723} 724 725extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) { 726 for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E; 727 ++GV) { 728 GV->setDoesNotThrow(); 729 Function *F = dyn_cast<Function>(GV); 730 if (F == nullptr) 731 continue; 732 733 for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) { 734 for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) { 735 if (isa<InvokeInst>(I)) { 736 InvokeInst *CI = cast<InvokeInst>(I); 737 CI->setDoesNotThrow(); 738 } 739 } 740 } 741 } 742} 743 744extern "C" void 745LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module, 746 LLVMTargetMachineRef TMR) { 747 TargetMachine *Target = unwrap(TMR); 748 unwrap(Module)->setDataLayout(Target->createDataLayout()); 749} 750 751extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) { 752 unwrap(M)->setPICLevel(PICLevel::Level::BigPIC); 753} 754 755extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { 756 unwrap(M)->setPIELevel(PIELevel::Level::Large); 757} 758 759// Here you'll find an implementation of ThinLTO as used by the Rust compiler 760// right now. This ThinLTO support is only enabled on "recent ish" versions of 761// LLVM, and otherwise it's just blanket rejected from other compilers. 762// 763// Most of this implementation is straight copied from LLVM. At the time of 764// this writing it wasn't *quite* suitable to reuse more code from upstream 765// for our purposes, but we should strive to upstream this support once it's 766// ready to go! I figure we may want a bit of testing locally first before 767// sending this upstream to LLVM. I hear though they're quite eager to receive 768// feedback like this! 769// 770// If you're reading this code and wondering "what in the world" or you're 771// working "good lord by LLVM upgrade is *still* failing due to these bindings" 772// then fear not! (ok maybe fear a little). All code here is mostly based 773// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM. 774// 775// You'll find that the general layout here roughly corresponds to the `run` 776// method in that file as well as `ProcessThinLTOModule`. Functions are 777// specifically commented below as well, but if you're updating this code 778// or otherwise trying to understand it, the LLVM source will be useful in 779// interpreting the mysteries within. 780// 781// Otherwise I'll apologize in advance, it probably requires a relatively 782// significant investment on your part to "truly understand" what's going on 783// here. Not saying I do myself, but it took me awhile staring at LLVM's source 784// and various online resources about ThinLTO to make heads or tails of all 785// this. 786 787// This is a shared data structure which *must* be threadsafe to share 788// read-only amongst threads. This also corresponds basically to the arguments 789// of the `ProcessThinLTOModule` function in the LLVM source. 790struct LLVMRustThinLTOData { 791 // The combined index that is the global analysis over all modules we're 792 // performing ThinLTO for. This is mostly managed by LLVM. 793 ModuleSummaryIndex Index; 794 795 // All modules we may look at, stored as in-memory serialized versions. This 796 // is later used when inlining to ensure we can extract any module to inline 797 // from. 798 StringMap<MemoryBufferRef> ModuleMap; 799 800 // A set that we manage of everything we *don't* want internalized. Note that 801 // this includes all transitive references right now as well, but it may not 802 // always! 803 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols; 804 805 // Not 100% sure what these are, but they impact what's internalized and 806 // what's inlined across modules, I believe. 807 StringMap<FunctionImporter::ImportMapTy> ImportLists; 808 StringMap<FunctionImporter::ExportSetTy> ExportLists; 809 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries; 810 811 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} 812}; 813 814// Just an argument to the `LLVMRustCreateThinLTOData` function below. 815struct LLVMRustThinLTOModule { 816 const char *identifier; 817 const char *data; 818 size_t len; 819}; 820 821// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it 822// does. 823static const GlobalValueSummary * 824getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { 825 auto StrongDefForLinker = llvm::find_if( 826 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) { 827 auto Linkage = Summary->linkage(); 828 return !GlobalValue::isAvailableExternallyLinkage(Linkage) && 829 !GlobalValue::isWeakForLinker(Linkage); 830 }); 831 if (StrongDefForLinker != GVSummaryList.end()) 832 return StrongDefForLinker->get(); 833 834 auto FirstDefForLinker = llvm::find_if( 835 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) { 836 auto Linkage = Summary->linkage(); 837 return !GlobalValue::isAvailableExternallyLinkage(Linkage); 838 }); 839 if (FirstDefForLinker == GVSummaryList.end()) 840 return nullptr; 841 return FirstDefForLinker->get(); 842} 843 844// The main entry point for creating the global ThinLTO analysis. The structure 845// here is basically the same as before threads are spawned in the `run` 846// function of `lib/LTO/ThinLTOCodeGenerator.cpp`. 847extern "C" LLVMRustThinLTOData* 848LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, 849 int num_modules, 850 const char **preserved_symbols, 851 int num_symbols) { 852 auto Ret = llvm::make_unique<LLVMRustThinLTOData>(); 853 854 // Load each module's summary and merge it into one combined index 855 for (int i = 0; i < num_modules; i++) { 856 auto module = &modules[i]; 857 StringRef buffer(module->data, module->len); 858 MemoryBufferRef mem_buffer(buffer, module->identifier); 859 860 Ret->ModuleMap[module->identifier] = mem_buffer; 861 862 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { 863 LLVMRustSetLastError(toString(std::move(Err)).c_str()); 864 return nullptr; 865 } 866 } 867 868 // Collect for each module the list of function it defines (GUID -> Summary) 869 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries); 870 871 // Convert the preserved symbols set from string to GUID, this is then needed 872 // for internalization. 873 for (int i = 0; i < num_symbols; i++) { 874 auto GUID = GlobalValue::getGUID(preserved_symbols[i]); 875 Ret->GUIDPreservedSymbols.insert(GUID); 876 } 877 878 // Collect the import/export lists for all modules from the call-graph in the 879 // combined index 880 // 881 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` 882 auto deadIsPrevailing = [&](GlobalValue::GUID G) { 883 return PrevailingType::Unknown; 884 }; 885#if LLVM_VERSION_GE(8, 0) 886 // We don't have a complete picture in our use of ThinLTO, just our immediate 887 // crate, so we need `ImportEnabled = false` to limit internalization. 888 // Otherwise, we sometimes lose `static` values -- see #60184. 889 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, 890 deadIsPrevailing, /* ImportEnabled = */ false); 891#else 892 computeDeadSymbols(Ret->Index, Ret->GUIDPreservedSymbols, deadIsPrevailing); 893#endif 894 ComputeCrossModuleImport( 895 Ret->Index, 896 Ret->ModuleToDefinedGVSummaries, 897 Ret->ImportLists, 898 Ret->ExportLists 899 ); 900 901 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it 902 // impacts the caching. 903 // 904 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this 905 // being lifted from `lib/LTO/LTO.cpp` as well 906 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; 907 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; 908 for (auto &I : Ret->Index) { 909 if (I.second.SummaryList.size() > 1) 910 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList); 911 } 912 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { 913 const auto &Prevailing = PrevailingCopy.find(GUID); 914 if (Prevailing == PrevailingCopy.end()) 915 return true; 916 return Prevailing->second == S; 917 }; 918 auto recordNewLinkage = [&](StringRef ModuleIdentifier, 919 GlobalValue::GUID GUID, 920 GlobalValue::LinkageTypes NewLinkage) { 921 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; 922 }; 923#if LLVM_VERSION_GE(9, 0) 924 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage, 925 Ret->GUIDPreservedSymbols); 926#elif LLVM_VERSION_GE(8, 0) 927 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage); 928#else 929 thinLTOResolveWeakForLinkerInIndex(Ret->Index, isPrevailing, recordNewLinkage); 930#endif 931 932 // Here we calculate an `ExportedGUIDs` set for use in the `isExported` 933 // callback below. This callback below will dictate the linkage for all 934 // summaries in the index, and we basically just only want to ensure that dead 935 // symbols are internalized. Otherwise everything that's already external 936 // linkage will stay as external, and internal will stay as internal. 937 std::set<GlobalValue::GUID> ExportedGUIDs; 938 for (auto &List : Ret->Index) { 939 for (auto &GVS: List.second.SummaryList) { 940 if (GlobalValue::isLocalLinkage(GVS->linkage())) 941 continue; 942 auto GUID = GVS->getOriginalName(); 943 if (GVS->flags().Live) 944 ExportedGUIDs.insert(GUID); 945 } 946 } 947 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) { 948 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); 949 return (ExportList != Ret->ExportLists.end() && 950 ExportList->second.count(GUID)) || 951 ExportedGUIDs.count(GUID); 952 }; 953 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported); 954 955 return Ret.release(); 956} 957 958extern "C" void 959LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { 960 delete Data; 961} 962 963// Below are the various passes that happen *per module* when doing ThinLTO. 964// 965// In other words, these are the functions that are all run concurrently 966// with one another, one per module. The passes here correspond to the analysis 967// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the 968// `ProcessThinLTOModule` function. Here they're split up into separate steps 969// so rustc can save off the intermediate bytecode between each step. 970 971extern "C" bool 972LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 973 Module &Mod = *unwrap(M); 974 if (renameModuleForThinLTO(Mod, Data->Index)) { 975 LLVMRustSetLastError("renameModuleForThinLTO failed"); 976 return false; 977 } 978 return true; 979} 980 981extern "C" bool 982LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 983 Module &Mod = *unwrap(M); 984 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); 985#if LLVM_VERSION_GE(8, 0) 986 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals); 987#else 988 thinLTOResolveWeakForLinkerModule(Mod, DefinedGlobals); 989#endif 990 return true; 991} 992 993extern "C" bool 994LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 995 Module &Mod = *unwrap(M); 996 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); 997 thinLTOInternalizeModule(Mod, DefinedGlobals); 998 return true; 999} 1000 1001extern "C" bool 1002LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 1003 Module &Mod = *unwrap(M); 1004 1005 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier()); 1006 auto Loader = [&](StringRef Identifier) { 1007 const auto &Memory = Data->ModuleMap.lookup(Identifier); 1008 auto &Context = Mod.getContext(); 1009 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true); 1010 1011 if (!MOrErr) 1012 return MOrErr; 1013 1014 // The rest of this closure is a workaround for 1015 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports 1016 // we accidentally import wasm custom sections into different modules, 1017 // duplicating them by in the final output artifact. 1018 // 1019 // The issue is worked around here by manually removing the 1020 // `wasm.custom_sections` named metadata node from any imported module. This 1021 // we know isn't used by any optimization pass so there's no need for it to 1022 // be imported. 1023 // 1024 // Note that the metadata is currently lazily loaded, so we materialize it 1025 // here before looking up if there's metadata inside. The `FunctionImporter` 1026 // will immediately materialize metadata anyway after an import, so this 1027 // shouldn't be a perf hit. 1028 if (Error Err = (*MOrErr)->materializeMetadata()) { 1029 Expected<std::unique_ptr<Module>> Ret(std::move(Err)); 1030 return Ret; 1031 } 1032 1033 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections"); 1034 if (WasmCustomSections) 1035 WasmCustomSections->eraseFromParent(); 1036 1037 return MOrErr; 1038 }; 1039 FunctionImporter Importer(Data->Index, Loader); 1040 Expected<bool> Result = Importer.importFunctions(Mod, ImportList); 1041 if (!Result) { 1042 LLVMRustSetLastError(toString(Result.takeError()).c_str()); 1043 return false; 1044 } 1045 return true; 1046} 1047 1048extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload 1049 const char*, // importing module name 1050 const char*); // imported module name 1051 1052// Calls `module_name_callback` for each module import done by ThinLTO. 1053// The callback is provided with regular null-terminated C strings. 1054extern "C" void 1055LLVMRustGetThinLTOModuleImports(const LLVMRustThinLTOData *data, 1056 LLVMRustModuleNameCallback module_name_callback, 1057 void* callback_payload) { 1058 for (const auto& importing_module : data->ImportLists) { 1059 const std::string importing_module_id = importing_module.getKey().str(); 1060 const auto& imports = importing_module.getValue(); 1061 for (const auto& imported_module : imports) { 1062 const std::string imported_module_id = imported_module.getKey().str(); 1063 module_name_callback(callback_payload, 1064 importing_module_id.c_str(), 1065 imported_module_id.c_str()); 1066 } 1067 } 1068} 1069 1070// This struct and various functions are sort of a hack right now, but the 1071// problem is that we've got in-memory LLVM modules after we generate and 1072// optimize all codegen-units for one compilation in rustc. To be compatible 1073// with the LTO support above we need to serialize the modules plus their 1074// ThinLTO summary into memory. 1075// 1076// This structure is basically an owned version of a serialize module, with 1077// a ThinLTO summary attached. 1078struct LLVMRustThinLTOBuffer { 1079 std::string data; 1080}; 1081 1082extern "C" LLVMRustThinLTOBuffer* 1083LLVMRustThinLTOBufferCreate(LLVMModuleRef M) { 1084 auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>(); 1085 { 1086 raw_string_ostream OS(Ret->data); 1087 { 1088 legacy::PassManager PM; 1089 PM.add(createWriteThinLTOBitcodePass(OS)); 1090 PM.run(*unwrap(M)); 1091 } 1092 } 1093 return Ret.release(); 1094} 1095 1096extern "C" void 1097LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { 1098 delete Buffer; 1099} 1100 1101extern "C" const void* 1102LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { 1103 return Buffer->data.data(); 1104} 1105 1106extern "C" size_t 1107LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { 1108 return Buffer->data.length(); 1109} 1110 1111// This is what we used to parse upstream bitcode for actual ThinLTO 1112// processing. We'll call this once per module optimized through ThinLTO, and 1113// it'll be called concurrently on many threads. 1114extern "C" LLVMModuleRef 1115LLVMRustParseBitcodeForLTO(LLVMContextRef Context, 1116 const char *data, 1117 size_t len, 1118 const char *identifier) { 1119 StringRef Data(data, len); 1120 MemoryBufferRef Buffer(Data, identifier); 1121 unwrap(Context)->enableDebugTypeODRUniquing(); 1122 Expected<std::unique_ptr<Module>> SrcOrError = 1123 parseBitcodeFile(Buffer, *unwrap(Context)); 1124 if (!SrcOrError) { 1125 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); 1126 return nullptr; 1127 } 1128 return wrap(std::move(*SrcOrError).release()); 1129} 1130 1131// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See 1132// the comment in `back/lto.rs` for why this exists. 1133extern "C" void 1134LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod, 1135 DICompileUnit **A, 1136 DICompileUnit **B) { 1137 Module *M = unwrap(Mod); 1138 DICompileUnit **Cur = A; 1139 DICompileUnit **Next = B; 1140 for (DICompileUnit *CU : M->debug_compile_units()) { 1141 *Cur = CU; 1142 Cur = Next; 1143 Next = nullptr; 1144 if (Cur == nullptr) 1145 break; 1146 } 1147} 1148 1149// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See 1150// the comment in `back/lto.rs` for why this exists. 1151extern "C" void 1152LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) { 1153 Module *M = unwrap(Mod); 1154 1155 // If the original source module didn't have a `DICompileUnit` then try to 1156 // merge all the existing compile units. If there aren't actually any though 1157 // then there's not much for us to do so return. 1158 if (Unit == nullptr) { 1159 for (DICompileUnit *CU : M->debug_compile_units()) { 1160 Unit = CU; 1161 break; 1162 } 1163 if (Unit == nullptr) 1164 return; 1165 } 1166 1167 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and 1168 // process it recursively. Note that we specifically iterate over instructions 1169 // to ensure we feed everything into it. 1170 DebugInfoFinder Finder; 1171 Finder.processModule(*M); 1172 for (Function &F : M->functions()) { 1173 for (auto &FI : F) { 1174 for (Instruction &BI : FI) { 1175 if (auto Loc = BI.getDebugLoc()) 1176 Finder.processLocation(*M, Loc); 1177 if (auto DVI = dyn_cast<DbgValueInst>(&BI)) 1178 Finder.processValue(*M, DVI); 1179 if (auto DDI = dyn_cast<DbgDeclareInst>(&BI)) 1180 Finder.processDeclare(*M, DDI); 1181 } 1182 } 1183 } 1184 1185 // After we've found all our debuginfo, rewrite all subprograms to point to 1186 // the same `DICompileUnit`. 1187 for (auto &F : Finder.subprograms()) { 1188 F->replaceUnit(Unit); 1189 } 1190 1191 // Erase any other references to other `DICompileUnit` instances, the verifier 1192 // will later ensure that we don't actually have any other stale references to 1193 // worry about. 1194 auto *MD = M->getNamedMetadata("llvm.dbg.cu"); 1195 MD->clearOperands(); 1196 MD->addOperand(Unit); 1197}