diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs
index 31da4522a1fda..1b8a69b8d93cf 100644
--- a/compiler/rustc_const_eval/src/interpret/place.rs
+++ b/compiler/rustc_const_eval/src/interpret/place.rs
@@ -277,7 +277,10 @@ where
         let layout = self.layout_of(pointee_type)?;
         let (ptr, meta) = match **val {
             Immediate::Scalar(ptr) => (ptr, MemPlaceMeta::None),
-            Immediate::ScalarPair(ptr, meta) => (ptr, MemPlaceMeta::Meta(meta.check_init()?)),
+            Immediate::ScalarPair(ptr, meta) => {
+                assert!(layout.is_unsized());
+                (ptr, MemPlaceMeta::Meta(meta.check_init()?))
+            }
         };
 
         let mplace = MemPlace {
@@ -298,8 +301,18 @@ where
         &self,
         src: &OpTy<'tcx, M::PointerTag>,
     ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::PointerTag>> {
-        let val = self.read_immediate(src)?;
-        trace!("deref to {} on {:?}", val.layout.ty, *val);
+        // HACK we need to special-case dereferencing a `Box` with a non-ZST allocator.
+        // See https://github.com/rust-lang/rust/issues/95453; codegen does similar things.
+        let src = if src.layout.ty.is_box() && !src.layout.field(self, 1).is_zst() {
+            // Extract `Box<T>` -> `Unique<T>` -> `NonNull<T>` -> `*const T`
+            let uniq = self.operand_field(src, 0)?;
+            let nonnull = self.operand_field(&uniq, 0)?;
+            self.operand_field(&nonnull, 0)?
+        } else {
+            *src
+        };
+        let val = self.read_immediate(&src)?;
+        debug!("deref to {} on {:?}", val.layout.ty, *val);
         let mplace = self.ref_to_mplace(&val)?;
         self.check_mplace_access(mplace, CheckInAllocMsg::DerefTest)?;
         Ok(mplace)
@@ -762,7 +775,8 @@ where
                     Abi::Scalar(_) => {} // fine
                     _ => span_bug!(
                         self.cur_span(),
-                        "write_immediate_to_mplace: invalid Scalar layout: {:#?}",
+                        "write_immediate_to_mplace: invalid layout for value {:?}: {:#?}",
+                        value,
                         dest.layout
                     ),
                 }
@@ -774,7 +788,8 @@ where
                 // which `ptr.offset(b_offset)` cannot possibly fail to satisfy.
                 let Abi::ScalarPair(a, b) = dest.layout.abi else { span_bug!(
                         self.cur_span(),
-                        "write_immediate_to_mplace: invalid ScalarPair layout: {:#?}",
+                        "write_immediate_to_mplace: invalid layout for value {:?}: {:#?}",
+                        value,
                         dest.layout
                     )
                 };
diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs
index 4a0aa41de739b..bc2f19677cf26 100644
--- a/compiler/rustc_const_eval/src/interpret/validity.rs
+++ b/compiler/rustc_const_eval/src/interpret/validity.rs
@@ -563,7 +563,9 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
                 self.check_safe_pointer(value, "reference")?;
                 Ok(true)
             }
-            ty::Adt(def, ..) if def.is_box() => {
+            // HACK: We only treat boxes with ZST allocators as primitives.
+            // See https://github.com/rust-lang/rust/issues/95453.
+            ty::Adt(def, ..) if def.is_box() && value.layout.field(self.ecx, 1).is_zst() => {
                 self.check_safe_pointer(value, "box")?;
                 Ok(true)
             }
@@ -786,7 +788,10 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
             return Ok(());
         }
         // Sanity check: `builtin_deref` does not know any pointers that are not primitive.
-        assert!(op.layout.ty.builtin_deref(true).is_none());
+        // FIXME: `builtin_deref` treats `Box` as a pointer even with arbitrary allocators,
+        // but we cannot treat that as a primitive, so we have to disable this.
+        // See https://github.com/rust-lang/rust/issues/95453.
+        // assert!(op.layout.ty.builtin_deref(true).is_none());
 
         // Special check preventing `UnsafeCell` in the inner part of constants
         if let Some(def) = op.layout.ty.ty_adt_def() {