fork/rust
git clone https://git.julia.blue/fork/rust
73528e339aa
1.40.0
1#include <stdio.h> 2 3#include <vector> 4#include <set> 5 6#include "rustllvm.h" 7 8#include "llvm/Analysis/TargetLibraryInfo.h" 9#include "llvm/Analysis/TargetTransformInfo.h" 10#include "llvm/CodeGen/TargetSubtargetInfo.h" 11#include "llvm/IR/AutoUpgrade.h" 12#include "llvm/IR/AssemblyAnnotationWriter.h" 13#include "llvm/IR/IntrinsicInst.h" 14#include "llvm/Support/CBindingWrapping.h" 15#include "llvm/Support/FileSystem.h" 16#include "llvm/Support/Host.h" 17#include "llvm/Target/TargetMachine.h" 18#include "llvm/Transforms/IPO/PassManagerBuilder.h" 19#include "llvm/Transforms/IPO/AlwaysInliner.h" 20#include "llvm/Transforms/IPO/FunctionImport.h" 21#include "llvm/Transforms/Utils/FunctionImportUtils.h" 22#include "llvm/LTO/LTO.h" 23 24#include "llvm-c/Transforms/PassManagerBuilder.h" 25 26using namespace llvm; 27using namespace llvm::legacy; 28 29typedef struct LLVMOpaquePass *LLVMPassRef; 30typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; 31 32DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef) 33DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) 34DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder, 35 LLVMPassManagerBuilderRef) 36 37extern "C" void LLVMInitializePasses() { 38 PassRegistry &Registry = *PassRegistry::getPassRegistry(); 39 initializeCore(Registry); 40 initializeCodeGen(Registry); 41 initializeScalarOpts(Registry); 42 initializeVectorization(Registry); 43 initializeIPO(Registry); 44 initializeAnalysis(Registry); 45 initializeTransformUtils(Registry); 46 initializeInstCombine(Registry); 47 initializeInstrumentation(Registry); 48 initializeTarget(Registry); 49} 50 51enum class LLVMRustPassKind { 52 Other, 53 Function, 54 Module, 55}; 56 57static LLVMRustPassKind toRust(PassKind Kind) { 58 switch (Kind) { 59 case PT_Function: 60 return LLVMRustPassKind::Function; 61 case PT_Module: 62 return LLVMRustPassKind::Module; 63 default: 64 return LLVMRustPassKind::Other; 65 } 66} 67 68extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) { 69 StringRef SR(PassName); 70 PassRegistry *PR = PassRegistry::getPassRegistry(); 71 72 const PassInfo *PI = PR->getPassInfo(SR); 73 if (PI) { 74 return wrap(PI->createPass()); 75 } 76 return nullptr; 77} 78 79extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) { 80 assert(RustPass); 81 Pass *Pass = unwrap(RustPass); 82 return toRust(Pass->getPassKind()); 83} 84 85extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) { 86 assert(RustPass); 87 Pass *Pass = unwrap(RustPass); 88 PassManagerBase *PMB = unwrap(PMR); 89 PMB->add(Pass); 90} 91 92extern "C" 93void LLVMRustPassManagerBuilderPopulateThinLTOPassManager( 94 LLVMPassManagerBuilderRef PMBR, 95 LLVMPassManagerRef PMR 96) { 97 unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR)); 98} 99 100extern "C" 101void LLVMRustAddLastExtensionPasses( 102 LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) { 103 auto AddExtensionPasses = [Passes, NumPasses]( 104 const PassManagerBuilder &Builder, PassManagerBase &PM) { 105 for (size_t I = 0; I < NumPasses; I++) { 106 PM.add(unwrap(Passes[I])); 107 } 108 }; 109 // Add the passes to both of the pre-finalization extension points, 110 // so they are run for optimized and non-optimized builds. 111 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast, 112 AddExtensionPasses); 113 unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, 114 AddExtensionPasses); 115} 116 117#ifdef LLVM_COMPONENT_X86 118#define SUBTARGET_X86 SUBTARGET(X86) 119#else 120#define SUBTARGET_X86 121#endif 122 123#ifdef LLVM_COMPONENT_ARM 124#define SUBTARGET_ARM SUBTARGET(ARM) 125#else 126#define SUBTARGET_ARM 127#endif 128 129#ifdef LLVM_COMPONENT_AARCH64 130#define SUBTARGET_AARCH64 SUBTARGET(AArch64) 131#else 132#define SUBTARGET_AARCH64 133#endif 134 135#ifdef LLVM_COMPONENT_MIPS 136#define SUBTARGET_MIPS SUBTARGET(Mips) 137#else 138#define SUBTARGET_MIPS 139#endif 140 141#ifdef LLVM_COMPONENT_POWERPC 142#define SUBTARGET_PPC SUBTARGET(PPC) 143#else 144#define SUBTARGET_PPC 145#endif 146 147#ifdef LLVM_COMPONENT_SYSTEMZ 148#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ) 149#else 150#define SUBTARGET_SYSTEMZ 151#endif 152 153#ifdef LLVM_COMPONENT_MSP430 154#define SUBTARGET_MSP430 SUBTARGET(MSP430) 155#else 156#define SUBTARGET_MSP430 157#endif 158 159#ifdef LLVM_COMPONENT_RISCV 160#define SUBTARGET_RISCV SUBTARGET(RISCV) 161#else 162#define SUBTARGET_RISCV 163#endif 164 165#ifdef LLVM_COMPONENT_SPARC 166#define SUBTARGET_SPARC SUBTARGET(Sparc) 167#else 168#define SUBTARGET_SPARC 169#endif 170 171#ifdef LLVM_COMPONENT_HEXAGON 172#define SUBTARGET_HEXAGON SUBTARGET(Hexagon) 173#else 174#define SUBTARGET_HEXAGON 175#endif 176 177#define GEN_SUBTARGETS \ 178 SUBTARGET_X86 \ 179 SUBTARGET_ARM \ 180 SUBTARGET_AARCH64 \ 181 SUBTARGET_MIPS \ 182 SUBTARGET_PPC \ 183 SUBTARGET_SYSTEMZ \ 184 SUBTARGET_MSP430 \ 185 SUBTARGET_SPARC \ 186 SUBTARGET_HEXAGON \ 187 SUBTARGET_RISCV \ 188 189#define SUBTARGET(x) \ 190 namespace llvm { \ 191 extern const SubtargetFeatureKV x##FeatureKV[]; \ 192 extern const SubtargetFeatureKV x##SubTypeKV[]; \ 193 } 194 195GEN_SUBTARGETS 196#undef SUBTARGET 197 198extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, 199 const char *Feature) { 200 TargetMachine *Target = unwrap(TM); 201 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 202 return MCInfo->checkFeatures(std::string("+") + Feature); 203} 204 205enum class LLVMRustCodeModel { 206 Other, 207 Small, 208 Kernel, 209 Medium, 210 Large, 211 None, 212}; 213 214static CodeModel::Model fromRust(LLVMRustCodeModel Model) { 215 switch (Model) { 216 case LLVMRustCodeModel::Small: 217 return CodeModel::Small; 218 case LLVMRustCodeModel::Kernel: 219 return CodeModel::Kernel; 220 case LLVMRustCodeModel::Medium: 221 return CodeModel::Medium; 222 case LLVMRustCodeModel::Large: 223 return CodeModel::Large; 224 default: 225 report_fatal_error("Bad CodeModel."); 226 } 227} 228 229enum class LLVMRustCodeGenOptLevel { 230 Other, 231 None, 232 Less, 233 Default, 234 Aggressive, 235}; 236 237static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) { 238 switch (Level) { 239 case LLVMRustCodeGenOptLevel::None: 240 return CodeGenOpt::None; 241 case LLVMRustCodeGenOptLevel::Less: 242 return CodeGenOpt::Less; 243 case LLVMRustCodeGenOptLevel::Default: 244 return CodeGenOpt::Default; 245 case LLVMRustCodeGenOptLevel::Aggressive: 246 return CodeGenOpt::Aggressive; 247 default: 248 report_fatal_error("Bad CodeGenOptLevel."); 249 } 250} 251 252enum class LLVMRustRelocMode { 253 Default, 254 Static, 255 PIC, 256 DynamicNoPic, 257 ROPI, 258 RWPI, 259 ROPIRWPI, 260}; 261 262static Optional<Reloc::Model> fromRust(LLVMRustRelocMode RustReloc) { 263 switch (RustReloc) { 264 case LLVMRustRelocMode::Default: 265 return None; 266 case LLVMRustRelocMode::Static: 267 return Reloc::Static; 268 case LLVMRustRelocMode::PIC: 269 return Reloc::PIC_; 270 case LLVMRustRelocMode::DynamicNoPic: 271 return Reloc::DynamicNoPIC; 272 case LLVMRustRelocMode::ROPI: 273 return Reloc::ROPI; 274 case LLVMRustRelocMode::RWPI: 275 return Reloc::RWPI; 276 case LLVMRustRelocMode::ROPIRWPI: 277 return Reloc::ROPI_RWPI; 278 } 279 report_fatal_error("Bad RelocModel."); 280} 281 282#ifdef LLVM_RUSTLLVM 283/// getLongestEntryLength - Return the length of the longest entry in the table. 284template<typename KV> 285static size_t getLongestEntryLength(ArrayRef<KV> Table) { 286 size_t MaxLen = 0; 287 for (auto &I : Table) 288 MaxLen = std::max(MaxLen, std::strlen(I.Key)); 289 return MaxLen; 290} 291 292extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) { 293 const TargetMachine *Target = unwrap(TM); 294 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 295 const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch(); 296 const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); 297 const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable(); 298 unsigned MaxCPULen = getLongestEntryLength(CPUTable); 299 300 printf("Available CPUs for this target:\n"); 301 if (HostArch == TargetArch) { 302 const StringRef HostCPU = sys::getHostCPUName(); 303 printf(" %-*s - Select the CPU of the current host (currently %.*s).\n", 304 MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data()); 305 } 306 for (auto &CPU : CPUTable) 307 printf(" %-*s\n", MaxCPULen, CPU.Key); 308 printf("\n"); 309} 310 311extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef TM) { 312 const TargetMachine *Target = unwrap(TM); 313 const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); 314 const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable(); 315 unsigned MaxFeatLen = getLongestEntryLength(FeatTable); 316 317 printf("Available features for this target:\n"); 318 for (auto &Feature : FeatTable) 319 printf(" %-*s - %s.\n", MaxFeatLen, Feature.Key, Feature.Desc); 320 printf("\n"); 321 322 printf("Use +feature to enable a feature, or -feature to disable it.\n" 323 "For example, rustc -C -target-cpu=mycpu -C " 324 "target-feature=+feature1,-feature2\n\n"); 325} 326 327#else 328 329extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) { 330 printf("Target CPU help is not supported by this LLVM version.\n\n"); 331} 332 333extern "C" void LLVMRustPrintTargetFeatures(LLVMTargetMachineRef) { 334 printf("Target features help is not supported by this LLVM version.\n\n"); 335} 336#endif 337 338extern "C" const char* LLVMRustGetHostCPUName(size_t *len) { 339 StringRef Name = sys::getHostCPUName(); 340 *len = Name.size(); 341 return Name.data(); 342} 343 344extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( 345 const char *TripleStr, const char *CPU, const char *Feature, 346 LLVMRustCodeModel RustCM, LLVMRustRelocMode RustReloc, 347 LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, 348 bool PositionIndependentExecutable, bool FunctionSections, 349 bool DataSections, 350 bool TrapUnreachable, 351 bool Singlethread, 352 bool AsmComments, 353 bool EmitStackSizeSection) { 354 355 auto OptLevel = fromRust(RustOptLevel); 356 auto RM = fromRust(RustReloc); 357 358 std::string Error; 359 Triple Trip(Triple::normalize(TripleStr)); 360 const llvm::Target *TheTarget = 361 TargetRegistry::lookupTarget(Trip.getTriple(), Error); 362 if (TheTarget == nullptr) { 363 LLVMRustSetLastError(Error.c_str()); 364 return nullptr; 365 } 366 367 TargetOptions Options; 368 369 Options.FloatABIType = FloatABI::Default; 370 if (UseSoftFloat) { 371 Options.FloatABIType = FloatABI::Soft; 372 } 373 Options.DataSections = DataSections; 374 Options.FunctionSections = FunctionSections; 375 Options.MCOptions.AsmVerbose = AsmComments; 376 Options.MCOptions.PreserveAsmComments = AsmComments; 377 378 if (TrapUnreachable) { 379 // Tell LLVM to codegen `unreachable` into an explicit trap instruction. 380 // This limits the extent of possible undefined behavior in some cases, as 381 // it prevents control flow from "falling through" into whatever code 382 // happens to be laid out next in memory. 383 Options.TrapUnreachable = true; 384 } 385 386 if (Singlethread) { 387 Options.ThreadModel = ThreadModel::Single; 388 } 389 390 Options.EmitStackSizeSection = EmitStackSizeSection; 391 392 Optional<CodeModel::Model> CM; 393 if (RustCM != LLVMRustCodeModel::None) 394 CM = fromRust(RustCM); 395 TargetMachine *TM = TheTarget->createTargetMachine( 396 Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); 397 return wrap(TM); 398} 399 400extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { 401 delete unwrap(TM); 402} 403 404// Unfortunately, LLVM doesn't expose a C API to add the corresponding analysis 405// passes for a target to a pass manager. We export that functionality through 406// this function. 407extern "C" void LLVMRustAddAnalysisPasses(LLVMTargetMachineRef TM, 408 LLVMPassManagerRef PMR, 409 LLVMModuleRef M) { 410 PassManagerBase *PM = unwrap(PMR); 411 PM->add( 412 createTargetTransformInfoWrapperPass(unwrap(TM)->getTargetIRAnalysis())); 413} 414 415extern "C" void LLVMRustConfigurePassManagerBuilder( 416 LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel, 417 bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO, 418 const char* PGOGenPath, const char* PGOUsePath) { 419#if LLVM_VERSION_GE(7, 0) 420 unwrap(PMBR)->MergeFunctions = MergeFunctions; 421#endif 422 unwrap(PMBR)->SLPVectorize = SLPVectorize; 423 unwrap(PMBR)->OptLevel = fromRust(OptLevel); 424 unwrap(PMBR)->LoopVectorize = LoopVectorize; 425 unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO; 426 427 if (PGOGenPath) { 428 assert(!PGOUsePath); 429 unwrap(PMBR)->EnablePGOInstrGen = true; 430 unwrap(PMBR)->PGOInstrGen = PGOGenPath; 431 } 432 if (PGOUsePath) { 433 assert(!PGOGenPath); 434 unwrap(PMBR)->PGOInstrUse = PGOUsePath; 435 } 436} 437 438// Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo` 439// field of a PassManagerBuilder, we expose our own method of doing so. 440extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR, 441 LLVMModuleRef M, 442 bool DisableSimplifyLibCalls) { 443 Triple TargetTriple(unwrap(M)->getTargetTriple()); 444 TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple); 445 if (DisableSimplifyLibCalls) 446 TLI->disableAllFunctions(); 447 unwrap(PMBR)->LibraryInfo = TLI; 448} 449 450// Unfortunately, the LLVM C API doesn't provide a way to create the 451// TargetLibraryInfo pass, so we use this method to do so. 452extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M, 453 bool DisableSimplifyLibCalls) { 454 Triple TargetTriple(unwrap(M)->getTargetTriple()); 455 TargetLibraryInfoImpl TLII(TargetTriple); 456 if (DisableSimplifyLibCalls) 457 TLII.disableAllFunctions(); 458 unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII)); 459} 460 461// Unfortunately, the LLVM C API doesn't provide an easy way of iterating over 462// all the functions in a module, so we do that manually here. You'll find 463// similar code in clang's BackendUtil.cpp file. 464extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR, 465 LLVMModuleRef M) { 466 llvm::legacy::FunctionPassManager *P = 467 unwrap<llvm::legacy::FunctionPassManager>(PMR); 468 P->doInitialization(); 469 470 // Upgrade all calls to old intrinsics first. 471 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;) 472 UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove 473 474 for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E; 475 ++I) 476 if (!I->isDeclaration()) 477 P->run(*I); 478 479 P->doFinalization(); 480} 481 482extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) { 483 // Initializing the command-line options more than once is not allowed. So, 484 // check if they've already been initialized. (This could happen if we're 485 // being called from rustpkg, for example). If the arguments change, then 486 // that's just kinda unfortunate. 487 static bool Initialized = false; 488 if (Initialized) 489 return; 490 Initialized = true; 491 cl::ParseCommandLineOptions(Argc, Argv); 492} 493 494enum class LLVMRustFileType { 495 Other, 496 AssemblyFile, 497 ObjectFile, 498}; 499 500static TargetMachine::CodeGenFileType fromRust(LLVMRustFileType Type) { 501 switch (Type) { 502 case LLVMRustFileType::AssemblyFile: 503 return TargetMachine::CGFT_AssemblyFile; 504 case LLVMRustFileType::ObjectFile: 505 return TargetMachine::CGFT_ObjectFile; 506 default: 507 report_fatal_error("Bad FileType."); 508 } 509} 510 511extern "C" LLVMRustResult 512LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, 513 LLVMModuleRef M, const char *Path, 514 LLVMRustFileType RustFileType) { 515 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR); 516 auto FileType = fromRust(RustFileType); 517 518 std::string ErrorInfo; 519 std::error_code EC; 520 raw_fd_ostream OS(Path, EC, sys::fs::F_None); 521 if (EC) 522 ErrorInfo = EC.message(); 523 if (ErrorInfo != "") { 524 LLVMRustSetLastError(ErrorInfo.c_str()); 525 return LLVMRustResult::Failure; 526 } 527 528#if LLVM_VERSION_GE(7, 0) 529 buffer_ostream BOS(OS); 530 unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false); 531#else 532 unwrap(Target)->addPassesToEmitFile(*PM, OS, FileType, false); 533#endif 534 PM->run(*unwrap(M)); 535 536 // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output 537 // stream (OS), so the only real safe place to delete this is here? Don't we 538 // wish this was written in Rust? 539 delete PM; 540 return LLVMRustResult::Success; 541} 542 543 544// Callback to demangle function name 545// Parameters: 546// * name to be demangled 547// * name len 548// * output buffer 549// * output buffer len 550// Returns len of demangled string, or 0 if demangle failed. 551typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t); 552 553 554namespace { 555 556class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter { 557 DemangleFn Demangle; 558 std::vector<char> Buf; 559 560public: 561 RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {} 562 563 // Return empty string if demangle failed 564 // or if name does not need to be demangled 565 StringRef CallDemangle(StringRef name) { 566 if (!Demangle) { 567 return StringRef(); 568 } 569 570 if (Buf.size() < name.size() * 2) { 571 // Semangled name usually shorter than mangled, 572 // but allocate twice as much memory just in case 573 Buf.resize(name.size() * 2); 574 } 575 576 auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size()); 577 if (!R) { 578 // Demangle failed. 579 return StringRef(); 580 } 581 582 auto Demangled = StringRef(Buf.data(), R); 583 if (Demangled == name) { 584 // Do not print anything if demangled name is equal to mangled. 585 return StringRef(); 586 } 587 588 return Demangled; 589 } 590 591 void emitFunctionAnnot(const Function *F, 592 formatted_raw_ostream &OS) override { 593 StringRef Demangled = CallDemangle(F->getName()); 594 if (Demangled.empty()) { 595 return; 596 } 597 598 OS << "; " << Demangled << "\n"; 599 } 600 601 void emitInstructionAnnot(const Instruction *I, 602 formatted_raw_ostream &OS) override { 603 const char *Name; 604 const Value *Value; 605 if (const CallInst *CI = dyn_cast<CallInst>(I)) { 606 Name = "call"; 607 Value = CI->getCalledValue(); 608 } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) { 609 Name = "invoke"; 610 Value = II->getCalledValue(); 611 } else { 612 // Could demangle more operations, e. g. 613 // `store %place, @function`. 614 return; 615 } 616 617 if (!Value->hasName()) { 618 return; 619 } 620 621 StringRef Demangled = CallDemangle(Value->getName()); 622 if (Demangled.empty()) { 623 return; 624 } 625 626 OS << "; " << Name << " " << Demangled << "\n"; 627 } 628}; 629 630class RustPrintModulePass : public ModulePass { 631 raw_ostream* OS; 632 DemangleFn Demangle; 633public: 634 static char ID; 635 RustPrintModulePass() : ModulePass(ID), OS(nullptr), Demangle(nullptr) {} 636 RustPrintModulePass(raw_ostream &OS, DemangleFn Demangle) 637 : ModulePass(ID), OS(&OS), Demangle(Demangle) {} 638 639 bool runOnModule(Module &M) override { 640 RustAssemblyAnnotationWriter AW(Demangle); 641 642 M.print(*OS, &AW, false); 643 644 return false; 645 } 646 647 void getAnalysisUsage(AnalysisUsage &AU) const override { 648 AU.setPreservesAll(); 649 } 650 651 static StringRef name() { return "RustPrintModulePass"; } 652}; 653 654} // namespace 655 656namespace llvm { 657 void initializeRustPrintModulePassPass(PassRegistry&); 658} 659 660char RustPrintModulePass::ID = 0; 661INITIALIZE_PASS(RustPrintModulePass, "print-rust-module", 662 "Print rust module to stderr", false, false) 663 664extern "C" LLVMRustResult 665LLVMRustPrintModule(LLVMPassManagerRef PMR, LLVMModuleRef M, 666 const char *Path, DemangleFn Demangle) { 667 llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR); 668 std::string ErrorInfo; 669 670 std::error_code EC; 671 raw_fd_ostream OS(Path, EC, sys::fs::F_None); 672 if (EC) 673 ErrorInfo = EC.message(); 674 if (ErrorInfo != "") { 675 LLVMRustSetLastError(ErrorInfo.c_str()); 676 return LLVMRustResult::Failure; 677 } 678 679 formatted_raw_ostream FOS(OS); 680 681 PM->add(new RustPrintModulePass(FOS, Demangle)); 682 683 PM->run(*unwrap(M)); 684 685 return LLVMRustResult::Success; 686} 687 688extern "C" void LLVMRustPrintPasses() { 689 LLVMInitializePasses(); 690 struct MyListener : PassRegistrationListener { 691 void passEnumerate(const PassInfo *Info) { 692 StringRef PassArg = Info->getPassArgument(); 693 StringRef PassName = Info->getPassName(); 694 if (!PassArg.empty()) { 695 // These unsigned->signed casts could theoretically overflow, but 696 // realistically never will (and even if, the result is implementation 697 // defined rather plain UB). 698 printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(), 699 (int)PassName.size(), PassName.data()); 700 } 701 } 702 } Listener; 703 704 PassRegistry *PR = PassRegistry::getPassRegistry(); 705 PR->enumerateWith(&Listener); 706} 707 708extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR, 709 bool AddLifetimes) { 710 unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes); 711} 712 713extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, 714 size_t Len) { 715 llvm::legacy::PassManager passes; 716 717 auto PreserveFunctions = [=](const GlobalValue &GV) { 718 for (size_t I = 0; I < Len; I++) { 719 if (GV.getName() == Symbols[I]) { 720 return true; 721 } 722 } 723 return false; 724 }; 725 726 passes.add(llvm::createInternalizePass(PreserveFunctions)); 727 728 passes.run(*unwrap(M)); 729} 730 731extern "C" void LLVMRustMarkAllFunctionsNounwind(LLVMModuleRef M) { 732 for (Module::iterator GV = unwrap(M)->begin(), E = unwrap(M)->end(); GV != E; 733 ++GV) { 734 GV->setDoesNotThrow(); 735 Function *F = dyn_cast<Function>(GV); 736 if (F == nullptr) 737 continue; 738 739 for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) { 740 for (BasicBlock::iterator I = B->begin(), IE = B->end(); I != IE; ++I) { 741 if (isa<InvokeInst>(I)) { 742 InvokeInst *CI = cast<InvokeInst>(I); 743 CI->setDoesNotThrow(); 744 } 745 } 746 } 747 } 748} 749 750extern "C" void 751LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module, 752 LLVMTargetMachineRef TMR) { 753 TargetMachine *Target = unwrap(TMR); 754 unwrap(Module)->setDataLayout(Target->createDataLayout()); 755} 756 757extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { 758 unwrap(M)->setPIELevel(PIELevel::Level::Large); 759} 760 761// Here you'll find an implementation of ThinLTO as used by the Rust compiler 762// right now. This ThinLTO support is only enabled on "recent ish" versions of 763// LLVM, and otherwise it's just blanket rejected from other compilers. 764// 765// Most of this implementation is straight copied from LLVM. At the time of 766// this writing it wasn't *quite* suitable to reuse more code from upstream 767// for our purposes, but we should strive to upstream this support once it's 768// ready to go! I figure we may want a bit of testing locally first before 769// sending this upstream to LLVM. I hear though they're quite eager to receive 770// feedback like this! 771// 772// If you're reading this code and wondering "what in the world" or you're 773// working "good lord by LLVM upgrade is *still* failing due to these bindings" 774// then fear not! (ok maybe fear a little). All code here is mostly based 775// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM. 776// 777// You'll find that the general layout here roughly corresponds to the `run` 778// method in that file as well as `ProcessThinLTOModule`. Functions are 779// specifically commented below as well, but if you're updating this code 780// or otherwise trying to understand it, the LLVM source will be useful in 781// interpreting the mysteries within. 782// 783// Otherwise I'll apologize in advance, it probably requires a relatively 784// significant investment on your part to "truly understand" what's going on 785// here. Not saying I do myself, but it took me awhile staring at LLVM's source 786// and various online resources about ThinLTO to make heads or tails of all 787// this. 788 789// This is a shared data structure which *must* be threadsafe to share 790// read-only amongst threads. This also corresponds basically to the arguments 791// of the `ProcessThinLTOModule` function in the LLVM source. 792struct LLVMRustThinLTOData { 793 // The combined index that is the global analysis over all modules we're 794 // performing ThinLTO for. This is mostly managed by LLVM. 795 ModuleSummaryIndex Index; 796 797 // All modules we may look at, stored as in-memory serialized versions. This 798 // is later used when inlining to ensure we can extract any module to inline 799 // from. 800 StringMap<MemoryBufferRef> ModuleMap; 801 802 // A set that we manage of everything we *don't* want internalized. Note that 803 // this includes all transitive references right now as well, but it may not 804 // always! 805 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols; 806 807 // Not 100% sure what these are, but they impact what's internalized and 808 // what's inlined across modules, I believe. 809 StringMap<FunctionImporter::ImportMapTy> ImportLists; 810 StringMap<FunctionImporter::ExportSetTy> ExportLists; 811 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries; 812 813#if LLVM_VERSION_GE(7, 0) 814 LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} 815#endif 816}; 817 818// Just an argument to the `LLVMRustCreateThinLTOData` function below. 819struct LLVMRustThinLTOModule { 820 const char *identifier; 821 const char *data; 822 size_t len; 823}; 824 825// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it 826// does. 827static const GlobalValueSummary * 828getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { 829 auto StrongDefForLinker = llvm::find_if( 830 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) { 831 auto Linkage = Summary->linkage(); 832 return !GlobalValue::isAvailableExternallyLinkage(Linkage) && 833 !GlobalValue::isWeakForLinker(Linkage); 834 }); 835 if (StrongDefForLinker != GVSummaryList.end()) 836 return StrongDefForLinker->get(); 837 838 auto FirstDefForLinker = llvm::find_if( 839 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) { 840 auto Linkage = Summary->linkage(); 841 return !GlobalValue::isAvailableExternallyLinkage(Linkage); 842 }); 843 if (FirstDefForLinker == GVSummaryList.end()) 844 return nullptr; 845 return FirstDefForLinker->get(); 846} 847 848// The main entry point for creating the global ThinLTO analysis. The structure 849// here is basically the same as before threads are spawned in the `run` 850// function of `lib/LTO/ThinLTOCodeGenerator.cpp`. 851extern "C" LLVMRustThinLTOData* 852LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, 853 int num_modules, 854 const char **preserved_symbols, 855 int num_symbols) { 856 auto Ret = llvm::make_unique<LLVMRustThinLTOData>(); 857 858 // Load each module's summary and merge it into one combined index 859 for (int i = 0; i < num_modules; i++) { 860 auto module = &modules[i]; 861 StringRef buffer(module->data, module->len); 862 MemoryBufferRef mem_buffer(buffer, module->identifier); 863 864 Ret->ModuleMap[module->identifier] = mem_buffer; 865 866 if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { 867 LLVMRustSetLastError(toString(std::move(Err)).c_str()); 868 return nullptr; 869 } 870 } 871 872 // Collect for each module the list of function it defines (GUID -> Summary) 873 Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries); 874 875 // Convert the preserved symbols set from string to GUID, this is then needed 876 // for internalization. 877 for (int i = 0; i < num_symbols; i++) { 878 auto GUID = GlobalValue::getGUID(preserved_symbols[i]); 879 Ret->GUIDPreservedSymbols.insert(GUID); 880 } 881 882 // Collect the import/export lists for all modules from the call-graph in the 883 // combined index 884 // 885 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` 886#if LLVM_VERSION_GE(7, 0) 887 auto deadIsPrevailing = [&](GlobalValue::GUID G) { 888 return PrevailingType::Unknown; 889 }; 890#if LLVM_VERSION_GE(8, 0) 891 // We don't have a complete picture in our use of ThinLTO, just our immediate 892 // crate, so we need `ImportEnabled = false` to limit internalization. 893 // Otherwise, we sometimes lose `static` values -- see #60184. 894 computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, 895 deadIsPrevailing, /* ImportEnabled = */ false); 896#else 897 computeDeadSymbols(Ret->Index, Ret->GUIDPreservedSymbols, deadIsPrevailing); 898#endif 899#else 900 computeDeadSymbols(Ret->Index, Ret->GUIDPreservedSymbols); 901#endif 902 ComputeCrossModuleImport( 903 Ret->Index, 904 Ret->ModuleToDefinedGVSummaries, 905 Ret->ImportLists, 906 Ret->ExportLists 907 ); 908 909 // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it 910 // impacts the caching. 911 // 912 // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this 913 // being lifted from `lib/LTO/LTO.cpp` as well 914 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; 915 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; 916 for (auto &I : Ret->Index) { 917 if (I.second.SummaryList.size() > 1) 918 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList); 919 } 920 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { 921 const auto &Prevailing = PrevailingCopy.find(GUID); 922 if (Prevailing == PrevailingCopy.end()) 923 return true; 924 return Prevailing->second == S; 925 }; 926 auto recordNewLinkage = [&](StringRef ModuleIdentifier, 927 GlobalValue::GUID GUID, 928 GlobalValue::LinkageTypes NewLinkage) { 929 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; 930 }; 931#if LLVM_VERSION_GE(9, 0) 932 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage, 933 Ret->GUIDPreservedSymbols); 934#elif LLVM_VERSION_GE(8, 0) 935 thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage); 936#else 937 thinLTOResolveWeakForLinkerInIndex(Ret->Index, isPrevailing, recordNewLinkage); 938#endif 939 940 // Here we calculate an `ExportedGUIDs` set for use in the `isExported` 941 // callback below. This callback below will dictate the linkage for all 942 // summaries in the index, and we basically just only want to ensure that dead 943 // symbols are internalized. Otherwise everything that's already external 944 // linkage will stay as external, and internal will stay as internal. 945 std::set<GlobalValue::GUID> ExportedGUIDs; 946 for (auto &List : Ret->Index) { 947 for (auto &GVS: List.second.SummaryList) { 948 if (GlobalValue::isLocalLinkage(GVS->linkage())) 949 continue; 950 auto GUID = GVS->getOriginalName(); 951 if (GVS->flags().Live) 952 ExportedGUIDs.insert(GUID); 953 } 954 } 955 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) { 956 const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); 957 return (ExportList != Ret->ExportLists.end() && 958 ExportList->second.count(GUID)) || 959 ExportedGUIDs.count(GUID); 960 }; 961 thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported); 962 963 return Ret.release(); 964} 965 966extern "C" void 967LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { 968 delete Data; 969} 970 971// Below are the various passes that happen *per module* when doing ThinLTO. 972// 973// In other words, these are the functions that are all run concurrently 974// with one another, one per module. The passes here correspond to the analysis 975// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the 976// `ProcessThinLTOModule` function. Here they're split up into separate steps 977// so rustc can save off the intermediate bytecode between each step. 978 979extern "C" bool 980LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 981 Module &Mod = *unwrap(M); 982 if (renameModuleForThinLTO(Mod, Data->Index)) { 983 LLVMRustSetLastError("renameModuleForThinLTO failed"); 984 return false; 985 } 986 return true; 987} 988 989extern "C" bool 990LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 991 Module &Mod = *unwrap(M); 992 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); 993#if LLVM_VERSION_GE(8, 0) 994 thinLTOResolvePrevailingInModule(Mod, DefinedGlobals); 995#else 996 thinLTOResolveWeakForLinkerModule(Mod, DefinedGlobals); 997#endif 998 return true; 999} 1000 1001extern "C" bool 1002LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 1003 Module &Mod = *unwrap(M); 1004 const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); 1005 thinLTOInternalizeModule(Mod, DefinedGlobals); 1006 return true; 1007} 1008 1009extern "C" bool 1010LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { 1011 Module &Mod = *unwrap(M); 1012 1013 const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier()); 1014 auto Loader = [&](StringRef Identifier) { 1015 const auto &Memory = Data->ModuleMap.lookup(Identifier); 1016 auto &Context = Mod.getContext(); 1017 auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true); 1018 1019 if (!MOrErr) 1020 return MOrErr; 1021 1022 // The rest of this closure is a workaround for 1023 // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports 1024 // we accidentally import wasm custom sections into different modules, 1025 // duplicating them by in the final output artifact. 1026 // 1027 // The issue is worked around here by manually removing the 1028 // `wasm.custom_sections` named metadata node from any imported module. This 1029 // we know isn't used by any optimization pass so there's no need for it to 1030 // be imported. 1031 // 1032 // Note that the metadata is currently lazily loaded, so we materialize it 1033 // here before looking up if there's metadata inside. The `FunctionImporter` 1034 // will immediately materialize metadata anyway after an import, so this 1035 // shouldn't be a perf hit. 1036 if (Error Err = (*MOrErr)->materializeMetadata()) { 1037 Expected<std::unique_ptr<Module>> Ret(std::move(Err)); 1038 return Ret; 1039 } 1040 1041 auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections"); 1042 if (WasmCustomSections) 1043 WasmCustomSections->eraseFromParent(); 1044 1045 return MOrErr; 1046 }; 1047 FunctionImporter Importer(Data->Index, Loader); 1048 Expected<bool> Result = Importer.importFunctions(Mod, ImportList); 1049 if (!Result) { 1050 LLVMRustSetLastError(toString(Result.takeError()).c_str()); 1051 return false; 1052 } 1053 return true; 1054} 1055 1056extern "C" typedef void (*LLVMRustModuleNameCallback)(void*, // payload 1057 const char*, // importing module name 1058 const char*); // imported module name 1059 1060// Calls `module_name_callback` for each module import done by ThinLTO. 1061// The callback is provided with regular null-terminated C strings. 1062extern "C" void 1063LLVMRustGetThinLTOModuleImports(const LLVMRustThinLTOData *data, 1064 LLVMRustModuleNameCallback module_name_callback, 1065 void* callback_payload) { 1066 for (const auto& importing_module : data->ImportLists) { 1067 const std::string importing_module_id = importing_module.getKey().str(); 1068 const auto& imports = importing_module.getValue(); 1069 for (const auto& imported_module : imports) { 1070 const std::string imported_module_id = imported_module.getKey().str(); 1071 module_name_callback(callback_payload, 1072 importing_module_id.c_str(), 1073 imported_module_id.c_str()); 1074 } 1075 } 1076} 1077 1078// This struct and various functions are sort of a hack right now, but the 1079// problem is that we've got in-memory LLVM modules after we generate and 1080// optimize all codegen-units for one compilation in rustc. To be compatible 1081// with the LTO support above we need to serialize the modules plus their 1082// ThinLTO summary into memory. 1083// 1084// This structure is basically an owned version of a serialize module, with 1085// a ThinLTO summary attached. 1086struct LLVMRustThinLTOBuffer { 1087 std::string data; 1088}; 1089 1090extern "C" LLVMRustThinLTOBuffer* 1091LLVMRustThinLTOBufferCreate(LLVMModuleRef M) { 1092 auto Ret = llvm::make_unique<LLVMRustThinLTOBuffer>(); 1093 { 1094 raw_string_ostream OS(Ret->data); 1095 { 1096 legacy::PassManager PM; 1097 PM.add(createWriteThinLTOBitcodePass(OS)); 1098 PM.run(*unwrap(M)); 1099 } 1100 } 1101 return Ret.release(); 1102} 1103 1104extern "C" void 1105LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { 1106 delete Buffer; 1107} 1108 1109extern "C" const void* 1110LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { 1111 return Buffer->data.data(); 1112} 1113 1114extern "C" size_t 1115LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { 1116 return Buffer->data.length(); 1117} 1118 1119// This is what we used to parse upstream bitcode for actual ThinLTO 1120// processing. We'll call this once per module optimized through ThinLTO, and 1121// it'll be called concurrently on many threads. 1122extern "C" LLVMModuleRef 1123LLVMRustParseBitcodeForLTO(LLVMContextRef Context, 1124 const char *data, 1125 size_t len, 1126 const char *identifier) { 1127 StringRef Data(data, len); 1128 MemoryBufferRef Buffer(Data, identifier); 1129 unwrap(Context)->enableDebugTypeODRUniquing(); 1130 Expected<std::unique_ptr<Module>> SrcOrError = 1131 parseBitcodeFile(Buffer, *unwrap(Context)); 1132 if (!SrcOrError) { 1133 LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); 1134 return nullptr; 1135 } 1136 return wrap(std::move(*SrcOrError).release()); 1137} 1138 1139// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See 1140// the comment in `back/lto.rs` for why this exists. 1141extern "C" void 1142LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod, 1143 DICompileUnit **A, 1144 DICompileUnit **B) { 1145 Module *M = unwrap(Mod); 1146 DICompileUnit **Cur = A; 1147 DICompileUnit **Next = B; 1148 for (DICompileUnit *CU : M->debug_compile_units()) { 1149 *Cur = CU; 1150 Cur = Next; 1151 Next = nullptr; 1152 if (Cur == nullptr) 1153 break; 1154 } 1155} 1156 1157// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See 1158// the comment in `back/lto.rs` for why this exists. 1159extern "C" void 1160LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) { 1161 Module *M = unwrap(Mod); 1162 1163 // If the original source module didn't have a `DICompileUnit` then try to 1164 // merge all the existing compile units. If there aren't actually any though 1165 // then there's not much for us to do so return. 1166 if (Unit == nullptr) { 1167 for (DICompileUnit *CU : M->debug_compile_units()) { 1168 Unit = CU; 1169 break; 1170 } 1171 if (Unit == nullptr) 1172 return; 1173 } 1174 1175 // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and 1176 // process it recursively. Note that we specifically iterate over instructions 1177 // to ensure we feed everything into it. 1178 DebugInfoFinder Finder; 1179 Finder.processModule(*M); 1180 for (Function &F : M->functions()) { 1181 for (auto &FI : F) { 1182 for (Instruction &BI : FI) { 1183 if (auto Loc = BI.getDebugLoc()) 1184 Finder.processLocation(*M, Loc); 1185 if (auto DVI = dyn_cast<DbgValueInst>(&BI)) 1186 Finder.processValue(*M, DVI); 1187 if (auto DDI = dyn_cast<DbgDeclareInst>(&BI)) 1188 Finder.processDeclare(*M, DDI); 1189 } 1190 } 1191 } 1192 1193 // After we've found all our debuginfo, rewrite all subprograms to point to 1194 // the same `DICompileUnit`. 1195 for (auto &F : Finder.subprograms()) { 1196 F->replaceUnit(Unit); 1197 } 1198 1199 // Erase any other references to other `DICompileUnit` instances, the verifier 1200 // will later ensure that we don't actually have any other stale references to 1201 // worry about. 1202 auto *MD = M->getNamedMetadata("llvm.dbg.cu"); 1203 MD->clearOperands(); 1204 MD->addOperand(Unit); 1205}