Merge rust-lang#81

81: Start serialising our own Yk IR. r=ltratt a=vext01



Co-authored-by: Edd Barrett <[email protected]>

Author: bors[bot]

llvm/include/llvm/CodeGen/AsmPrinter.h

@@ -661,6 +661,8 @@ class AsmPrinter : public MachineFunctionPass {
   /// Emit a long long directive and value.
   void emitInt64(uint64_t Value) const;
 
+  void emitSizeT(uint64_t Value) const;
+
   /// Emit something like ".long Hi-Lo" where the size in bytes of the directive
   /// is specified by Size and Hi/Lo specify the labels.  This implicitly uses
   /// .set if it is available.

llvm/include/llvm/MC/MCStreamer.h

@@ -743,6 +743,7 @@ class MCStreamer {
   void emitInt16(uint64_t Value) { emitIntValue(Value, 2); }
   void emitInt32(uint64_t Value) { emitIntValue(Value, 4); }
   void emitInt64(uint64_t Value) { emitIntValue(Value, 8); }
+  void emitSizeT(uint64_t Value) { emitIntValue(Value, sizeof(size_t)); }
 
   /// Special case of EmitValue that avoids the client having to pass
   /// in a MCExpr for constant integers & prints in Hex format for certain

llvm/include/llvm/YkIR/YkIRWriter.h

@@ -0,0 +1,12 @@
+#ifndef __YKIRWRITER_H
+#define __YKIRWRITER_H
+
+#include "llvm/IR/Module.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCStreamer.h"
+
+namespace llvm {
+  void embedYkIR(MCContext &Ctx, MCStreamer &OutStreamer, Module &M);
+} // namespace llvm
+
+#endif

llvm/lib/CMakeLists.txt

@@ -44,6 +44,7 @@ add_subdirectory(XRay)
 if (LLVM_INCLUDE_TESTS)
   add_subdirectory(Testing)
 endif()
+add_subdirectory(YkIR)
 add_subdirectory(WindowsDriver)
 add_subdirectory(WindowsManifest)
 

llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp

@@ -114,6 +114,7 @@
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
+#include "llvm/YkIR/YkIRWriter.h"
 #include <algorithm>
 #include <cassert>
 #include <cinttypes>
@@ -137,6 +138,7 @@ static cl::opt<bool>
 extern bool YkAllocLLVMBBAddrMapSection;
 extern bool YkExtendedLLVMBBAddrMapSection;
 extern bool YkStackMapOffsetFix;
+extern bool YkEmbedIR;
 
 const char DWARFGroupName[] = "dwarf";
 const char DWARFGroupDescription[] = "DWARF Emission";
@@ -2382,18 +2384,24 @@ bool AsmPrinter::doFinalization(Module &M) {
     OutStreamer->popSection();
   }
 
-  // The `embed-bitcode` flag serialises the IR after only architecture
+  // The `embed-bitcode` flag serialises the LLVM IR after only architecture
   // agnostic optimisations have been run, but then proceeds to apply other
   // optimisations and transformations afterwards. Sometimes this final version
-  // is precisely what we are interested in. The `embed-bitcode-final` flag
-  // waits until all optimisations/transformations have been run before
-  // embedding the IR.
+  // is precisely what we are interested in. The `embed-bitcode-final` and
+  // `yk-embed-ir` flags wait until all LLVM IR optimisations/transformations
+  // have been run before embedding their respective IR. Note that at the time
+  // of embedding, the MIR pipeline hasn't yet run, and this may do some more
+  // optimisation.
   if (EmbedBitcodeFinal)
     llvm::embedBitcodeInModule(M, llvm::MemoryBufferRef(),
                                /*EmbedBitcode*/ true,
                                /*EmbedCmdline*/ false,
                                /*CmdArgs*/ std::vector<uint8_t>());
 
+  if (YkEmbedIR) {
+    llvm::embedYkIR(OutContext, *OutStreamer, M);
+  }
+
   // Set the MachineFunction to nullptr so that we can catch attempted
   // accesses to MF specific features at the module level and so that
   // we can conditionalize accesses based on whether or not it is nullptr.

llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt

@@ -45,4 +45,5 @@ add_llvm_component_library(LLVMAsmPrinter
   Support
   Target
   TargetParser
+  YkIR
   )

llvm/lib/Support/Yk.cpp

@@ -111,6 +111,20 @@ struct CreateYkOptNoneAfterIRPassesParser {
 static ManagedStatic<cl::opt<bool, true>,
   CreateYkOptNoneAfterIRPassesParser> YkOptNoneAfterIRPassesParser;
 
+bool YkEmbedIR;
+namespace {
+struct CreateYkEmbedIRParser {
+  static void *call() {
+    return new cl::opt<bool, true>(
+        "yk-embed-ir",
+        cl::desc(
+            "Embed Yk IR into the binary."),
+        cl::NotHidden, cl::location(YkEmbedIR));
+  }
+};
+} // namespace
+static ManagedStatic<cl::opt<bool, true>, CreateYkEmbedIRParser> YkEmbedIRParser;
+
 void llvm::initYkOptions() {
   *YkAllocLLVMBCSectionParser;
   *YkAllocLLVMBBAddrMapSectionParser;
@@ -119,4 +133,5 @@ void llvm::initYkOptions() {
   *YkStackMapAdditionalLocsParser;
   *YkStackmapsSpillFixParser;
   *YkOptNoneAfterIRPassesParser;
+  *YkEmbedIRParser;
 }

llvm/lib/YkIR/CMakeLists.txt

@@ -0,0 +1,8 @@
+add_llvm_component_library(LLVMYkIR
+  YkIRWriter.cpp
+
+  LINK_COMPONENTS
+  Core
+  MC
+  Support
+  )

llvm/lib/YkIR/YkIRWriter.cpp

@@ -0,0 +1,126 @@
+//===- YkIR/YkIRWRiter.cpp -- Yk JIT IR Serialiaser---------------------===//
+//
+// Converts an LLVM module into Yk's on-disk AOT IR.
+//
+//===-------------------------------------------------------------------===//
+
+#include "llvm/ADT/Triple.h"
+#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Module.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCSectionELF.h"
+#include "llvm/MC/MCStreamer.h"
+
+using namespace llvm;
+using namespace std;
+
+namespace {
+
+const char *SectionName = ".yk_ir";
+const uint32_t Magic = 0xedd5f00d;
+const uint32_t Version = 0;
+
+enum OpCode {
+  Nop = 0,
+};
+
+class YkIRWriter {
+private:
+  Module &M;
+  MCStreamer &OutStreamer;
+
+public:
+  YkIRWriter(Module &M, MCStreamer &OutStreamer)
+      : M(M), OutStreamer(OutStreamer) {}
+
+  // Serialises a null-terminated string.
+  void serialiseString(StringRef S) {
+    OutStreamer.emitBinaryData(S);
+    OutStreamer.emitInt8(0); // null terminator.
+  }
+
+  void serialiseBlock(BasicBlock &BB) {
+    // num_instrs:
+    OutStreamer.emitSizeT(BB.size());
+    // instrs:
+    for (Instruction &I : BB) {
+      (void)I;
+      OutStreamer.emitInt8(OpCode::Nop);
+    }
+  }
+
+  void serialiseFunc(Function &F) {
+    // name:
+    serialiseString(F.getName());
+    // num_blocks:
+    OutStreamer.emitSizeT(F.size());
+    // blocks:
+    for (BasicBlock &BB : F) {
+      serialiseBlock(BB);
+    }
+  }
+
+  void serialise() {
+    // header:
+    OutStreamer.emitInt32(Magic);
+    OutStreamer.emitInt32(Version);
+
+    // num_funcs:
+    OutStreamer.emitSizeT(M.size());
+    // funcs:
+    for (Function &F : M) {
+      serialiseFunc(F);
+    }
+  }
+};
+} // anonymous namespace
+
+// Create an ELF section for storing Yk IR into.
+MCSection *createYkIRSection(MCContext &Ctx, const MCSection *TextSec) {
+  if (Ctx.getObjectFileType() != MCContext::IsELF)
+    return nullptr;
+
+  const MCSectionELF *ElfSec = static_cast<const MCSectionELF *>(TextSec);
+  unsigned Flags = ELF::SHF_LINK_ORDER;
+  StringRef GroupName;
+
+  // Ensure the loader loads it.
+  Flags |= ELF::SHF_ALLOC;
+
+  return Ctx.getELFSection(SectionName, ELF::SHT_LLVM_BB_ADDR_MAP, Flags, 0,
+                           GroupName, true, ElfSec->getUniqueID(),
+                           cast<MCSymbolELF>(TextSec->getBeginSymbol()));
+}
+
+// Emit a start/end IR marker.
+//
+// The JIT uses a start and end marker to make a Rust slice of the IR.
+void emitStartOrEndSymbol(MCContext &MCtxt, MCStreamer &OutStreamer,
+                          bool Start) {
+  std::string SymName("ykllvm.yk_ir.");
+  if (Start)
+    SymName.append("start");
+  else
+    SymName.append("stop");
+
+  MCSymbol *Sym = MCtxt.getOrCreateSymbol(SymName);
+  OutStreamer.emitSymbolAttribute(Sym, llvm::MCSA_Global);
+  OutStreamer.emitLabel(Sym);
+}
+
+namespace llvm {
+
+// Emit Yk IR into the resulting ELF binary.
+void embedYkIR(MCContext &Ctx, MCStreamer &OutStreamer, Module &M) {
+  MCSection *YkIRSec =
+      createYkIRSection(Ctx, std::get<0>(OutStreamer.getCurrentSection()));
+
+  OutStreamer.pushSection();
+  OutStreamer.switchSection(YkIRSec);
+  emitStartOrEndSymbol(Ctx, OutStreamer, true);
+  YkIRWriter(M, OutStreamer).serialise();
+  emitStartOrEndSymbol(Ctx, OutStreamer, false);
+  OutStreamer.popSection();
+}
+} // namespace llvm

yk_format_new_files.sh

@@ -13,7 +13,7 @@
 
 set -e
 
-YK_DIRS="./clang/test/Yk ./llvm/lib/Transforms/Yk ./llvm/include/llvm/Transforms/Yk"
+YK_DIRS="./clang/test/Yk ./llvm/lib/Transforms/Yk ./llvm/include/llvm/Transforms/Yk llvm/lib/YkIR"
 
 for dir in ${YK_DIRS}; do
     find ${dir} -type f -iname '*.cpp' -or -iname '*.h' -or -iname '*.c' | \