[IRGen] Emit null check before swift_conformsToProtocol for nullable metatypes

rdar://149882902

swift_conformsToProtocol does not properly handle nullptr values, which can currently be passed if the source type is an optional metatype. This change adds emission of a null check before calling the runtime function in these cases.

Author: drexin

lib/IRGen/GenCast.cpp

@@ -544,8 +544,8 @@ llvm::Value *irgen::emitMetatypeToAnyObjectDowncast(IRGenFunction &IGF,
 /// Emit a checked cast to a protocol or protocol composition.
 void irgen::emitScalarExistentialDowncast(
     IRGenFunction &IGF, llvm::Value *value, SILType srcType, SILType destType,
-    CheckedCastMode mode, std::optional<MetatypeRepresentation> metatypeKind,
-    Explosion &ex) {
+    CheckedCastMode mode, bool sourceWrappedInOptional,
+    std::optional<MetatypeRepresentation> metatypeKind, Explosion &ex) {
   auto srcInstanceType = srcType.getASTType();
   auto destInstanceType = destType.getASTType();
   while (auto metatypeType = dyn_cast<ExistentialMetatypeType>(
@@ -800,7 +800,36 @@ void irgen::emitScalarExistentialDowncast(
   for (auto proto : witnessTableProtos)
     args.push_back(proto);
 
-  auto valueAndWitnessTables = IGF.Builder.CreateCall(fn, args);
+  llvm::BasicBlock *nilCheckedCont = nullptr;
+  llvm::BasicBlock *nilBB = nullptr;
+  llvm::BasicBlock *nonNilBB = nullptr;
+  if (sourceWrappedInOptional) {
+    nilBB = IGF.createBasicBlock("is-nil");
+    nonNilBB = IGF.createBasicBlock("is-non-nil");
+    nilCheckedCont = IGF.createBasicBlock("nil-checked-cont");
+
+    auto isNotNil = IGF.Builder.CreateICmpNE(
+        metadataValue, llvm::ConstantPointerNull::get(
+                           cast<llvm::PointerType>(IGF.IGM.Int8PtrTy)));
+
+    IGF.Builder.CreateCondBr(isNotNil, nonNilBB, nilBB);
+    IGF.Builder.emitBlock(nilBB);
+    IGF.Builder.CreateBr(nilCheckedCont);
+    IGF.Builder.emitBlock(nonNilBB);
+  }
+
+  llvm::Value *valueAndWitnessTables = IGF.Builder.CreateCall(fn, args);
+
+  if (nilCheckedCont) {
+    IGF.Builder.CreateBr(nilCheckedCont);
+    IGF.Builder.emitBlock(nilCheckedCont);
+    auto *returnTy = valueAndWitnessTables->getType();
+    auto failureVal = llvm::Constant::getNullValue(returnTy);
+    auto phi = IGF.Builder.CreatePHI(returnTy, 2);
+    phi->addIncoming(valueAndWitnessTables, nonNilBB);
+    phi->addIncoming(failureVal, nilBB);
+    valueAndWitnessTables = phi;
+  }
 
   resultValue = IGF.Builder.CreateExtractValue(valueAndWitnessTables, 0);
   if (resultValue->getType() != resultType)
@@ -945,10 +974,9 @@ void irgen::emitScalarCheckedCast(IRGenFunction &IGF,
 
     // Casts to existential metatypes.
     if (auto existential = targetLoweredType.getAs<ExistentialMetatypeType>()) {
-      emitScalarExistentialDowncast(IGF, metatypeVal, sourceLoweredType,
-                                    targetLoweredType, mode,
-                                    existential->getRepresentation(),
-                                    out);
+      emitScalarExistentialDowncast(
+          IGF, metatypeVal, sourceLoweredType, targetLoweredType, mode,
+          sourceWrappedInOptional, existential->getRepresentation(), out);
       return;
 
     // Casts to concrete metatypes.
@@ -1023,9 +1051,10 @@ void irgen::emitScalarCheckedCast(IRGenFunction &IGF,
 
   if (targetFormalType.isExistentialType()) {
     Explosion outRes;
-    emitScalarExistentialDowncast(
-        IGF, instance, sourceLoweredType, targetLoweredType, mode,
-        /*not a metatype*/ std::nullopt, outRes);
+    emitScalarExistentialDowncast(IGF, instance, sourceLoweredType,
+                                  targetLoweredType, mode,
+                                  /*sourceWrappedInOptional*/ false,
+                                  /*not a metatype*/ std::nullopt, outRes);
     returnNilCheckedResult(IGF.Builder, outRes);
     return;
   }

lib/IRGen/GenCast.h

@@ -99,8 +99,8 @@ namespace irgen {
   /// not, the cast is done as a class instance cast.
   void emitScalarExistentialDowncast(
       IRGenFunction &IGF, llvm::Value *orig, SILType srcType, SILType destType,
-      CheckedCastMode mode, std::optional<MetatypeRepresentation> metatypeKind,
-      Explosion &ex);
+      CheckedCastMode mode, bool sourceWrappedInOptional,
+      std::optional<MetatypeRepresentation> metatypeKind, Explosion &ex);
 
   /// Emit a checked cast from a metatype to AnyObject.
   llvm::Value *emitMetatypeToAnyObjectDowncast(IRGenFunction &IGF,

test/IRGen/casts.sil

@@ -12,6 +12,7 @@ struct NotClass {}
 class A {}
 class B: A {}
 final class F: A {}
+protocol Q {}
 
 sil_vtable A {}
 sil_vtable B {}
@@ -435,3 +436,24 @@ bb2:
 bb3(%11 : $Optional<C & PAnyObject>):
   return %11 : $Optional<C & PAnyObject>
 }
+
+// CHECK-LABEL: define{{.*}} @checked_cast_optional_metatype
+// CHECK: [[COND:%.*]] = icmp ne ptr {{%.*}}, null
+// CHECK-NEXT: br i1 [[COND]], label %is-non-nil, label %is-nil
+// CHECK: nil-checked-cont:
+// CHECK:   %4 = phi { ptr, ptr } [ {{%.*}}, %is-non-nil ], [ zeroinitializer, %is-nil ]
+sil @checked_cast_optional_metatype : $@convention(thin) (Optional<@thick any Any.Type>) -> Optional<@thick any Q.Type> {
+bb0(%0 : $Optional<@thick any Any.Type>):
+  checked_cast_br Optional<any Any.Type> in %0 to any Q.Type, bb1, bb2
+
+bb1(%3 : $@thick any Q.Type):
+  %4 = enum $Optional<@thick any Q.Type>, #Optional.some!enumelt, %3
+  br bb3(%4)
+
+bb2:
+  %6 = enum $Optional<@thick any Q.Type>, #Optional.none!enumelt
+  br bb3(%6)
+
+bb3(%8 : $Optional<@thick any Q.Type>):
+  return %8
+}